1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/sysmacros.h> 28 #include <sys/ddi.h> 29 #include <sys/async.h> 30 #include <sys/sunddi.h> 31 #include <sys/ddifm.h> 32 #include <sys/fm/protocol.h> 33 #include <sys/vmem.h> 34 #include <sys/intr.h> 35 #include <sys/ivintr.h> 36 #include <sys/errno.h> 37 #include <sys/hypervisor_api.h> 38 #include <sys/hsvc.h> 39 #include <px_obj.h> 40 #include <sys/machsystm.h> 41 #include <sys/sunndi.h> 42 #include <sys/pcie_impl.h> 43 #include "px_lib4v.h" 44 #include "px_err.h" 45 #include <sys/pci_cfgacc.h> 46 #include <sys/pci_cfgacc_4v.h> 47 48 49 /* mask for the ranges property in calculating the real PFN range */ 50 uint_t px_ranges_phi_mask = ((1 << 28) -1); 51 52 /* 53 * Hypervisor VPCI services information for the px nexus driver. 54 */ 55 static uint64_t px_vpci_min_ver; /* Negotiated VPCI API minor version */ 56 static uint_t px_vpci_users = 0; /* VPCI API users */ 57 static hsvc_info_t px_hsvc_vpci = { 58 HSVC_REV_1, NULL, HSVC_GROUP_VPCI, PX_VPCI_MAJOR_VER, 59 PX_VPCI_MINOR_VER, "PX" 60 }; 61 62 /* 63 * Hypervisor SDIO services information for the px nexus driver. 64 */ 65 static uint64_t px_sdio_min_ver; /* Negotiated SDIO API minor version */ 66 static uint_t px_sdio_users = 0; /* SDIO API users */ 67 static hsvc_info_t px_hsvc_sdio = { 68 HSVC_REV_1, NULL, HSVC_GROUP_SDIO, PX_SDIO_MAJOR_VER, 69 PX_SDIO_MINOR_VER, "PX" 70 }; 71 72 /* 73 * Hypervisor SDIO ERR services information for the px nexus driver. 74 */ 75 static uint64_t px_sdio_err_min_ver; /* Negotiated SDIO ERR API */ 76 /* minor version */ 77 static uint_t px_sdio_err_users = 0; /* SDIO ERR API users */ 78 static hsvc_info_t px_hsvc_sdio_err = { 79 HSVC_REV_1, NULL, HSVC_GROUP_SDIO_ERR, PX_SDIO_ERR_MAJOR_VER, 80 PX_SDIO_ERR_MINOR_VER, "PX" 81 }; 82 83 #define CHILD_LOANED "child_loaned" 84 static int px_lib_count_waiting_dev(dev_info_t *); 85 86 int 87 px_lib_dev_init(dev_info_t *dip, devhandle_t *dev_hdl) 88 { 89 px_nexus_regspec_t *rp; 90 uint_t reglen; 91 int ret; 92 px_t *px_p = DIP_TO_STATE(dip); 93 uint64_t mjrnum; 94 uint64_t mnrnum; 95 96 DBG(DBG_ATTACH, dip, "px_lib_dev_init: dip 0x%p\n", dip); 97 98 /* 99 * Check HV intr group api versioning. 100 * This driver uses the old interrupt routines which are supported 101 * in old firmware in the CORE API group and in newer firmware in 102 * the INTR API group. Support for these calls will be dropped 103 * once the INTR API group major goes to 2. 104 */ 105 if ((hsvc_version(HSVC_GROUP_INTR, &mjrnum, &mnrnum) == 0) && 106 (mjrnum > 1)) { 107 cmn_err(CE_WARN, "px: unsupported intr api group: " 108 "maj:0x%lx, min:0x%lx", mjrnum, mnrnum); 109 return (ENOTSUP); 110 } 111 112 ret = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 113 "reg", (uchar_t **)&rp, ®len); 114 if (ret != DDI_PROP_SUCCESS) { 115 DBG(DBG_ATTACH, dip, "px_lib_dev_init failed ret=%d\n", ret); 116 return (DDI_FAILURE); 117 } 118 119 /* 120 * Initilize device handle. The device handle uniquely identifies 121 * a SUN4V device. It consists of the lower 28-bits of the hi-cell 122 * of the first entry of the SUN4V device's "reg" property as 123 * defined by the SUN4V Bus Binding to Open Firmware. 124 */ 125 *dev_hdl = (devhandle_t)((rp->phys_addr >> 32) & DEVHDLE_MASK); 126 ddi_prop_free(rp); 127 128 /* 129 * hotplug implementation requires this property to be associated with 130 * any indirect PCI config access services 131 */ 132 (void) ddi_prop_update_int(makedevice(ddi_driver_major(dip), 133 PCI_MINOR_NUM(ddi_get_instance(dip), PCI_DEVCTL_MINOR)), dip, 134 PCI_BUS_CONF_MAP_PROP, 1); 135 136 DBG(DBG_ATTACH, dip, "px_lib_dev_init: dev_hdl 0x%llx\n", *dev_hdl); 137 138 /* 139 * If a /pci node has a pci-intx-not-supported property, this property 140 * represents that the fabric doesn't support fixed interrupt. 141 */ 142 if (!ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 143 "pci-intx-not-supported")) { 144 DBG(DBG_ATTACH, dip, "px_lib_dev_init: " 145 "pci-intx-not-supported is not found, dip=0x%p\n", dip); 146 px_p->px_supp_intr_types |= DDI_INTR_TYPE_FIXED; 147 } 148 149 /* 150 * Negotiate the API version for VPCI hypervisor services. 151 */ 152 if ((px_vpci_users == 0) && 153 ((ret = hsvc_register(&px_hsvc_vpci, &px_vpci_min_ver)) != 0)) { 154 cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " 155 "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d\n", 156 px_hsvc_vpci.hsvc_modname, px_hsvc_vpci.hsvc_group, 157 px_hsvc_vpci.hsvc_major, px_hsvc_vpci.hsvc_minor, ret); 158 return (DDI_FAILURE); 159 } 160 px_vpci_users++; 161 DBG(DBG_ATTACH, dip, "px_lib_dev_init: negotiated VPCI API version, " 162 "major 0x%lx minor 0x%lx\n", px_hsvc_vpci.hsvc_major, 163 px_vpci_min_ver); 164 165 /* 166 * Negotiate the API version for SDIO hypervisor services. 167 */ 168 if ((px_sdio_users == 0) && 169 ((ret = hsvc_register(&px_hsvc_sdio, &px_sdio_min_ver)) != 0)) { 170 DBG(DBG_ATTACH, dip, "%s: cannot negotiate hypervisor " 171 "services group: 0x%lx major: 0x%lx minor: 0x%lx " 172 "errno: %d\n", px_hsvc_sdio.hsvc_modname, 173 px_hsvc_sdio.hsvc_group, px_hsvc_sdio.hsvc_major, 174 px_hsvc_sdio.hsvc_minor, ret); 175 } else { 176 px_sdio_users++; 177 DBG(DBG_ATTACH, dip, "px_lib_dev_init: negotiated SDIO API" 178 "version, major 0x%lx minor 0x%lx\n", 179 px_hsvc_sdio.hsvc_major, px_sdio_min_ver); 180 } 181 182 /* 183 * Negotiate the API version for SDIO ERR hypervisor services. 184 */ 185 if ((px_sdio_err_users == 0) && 186 ((ret = hsvc_register(&px_hsvc_sdio_err, 187 &px_sdio_err_min_ver)) != 0)) { 188 DBG(DBG_ATTACH, dip, "%s: cannot negotiate SDIO ERR hypervisor " 189 "services group: 0x%lx major: 0x%lx minor: 0x%lx " 190 "errno: %d\n", px_hsvc_sdio_err.hsvc_modname, 191 px_hsvc_sdio_err.hsvc_group, px_hsvc_sdio_err.hsvc_major, 192 px_hsvc_sdio_err.hsvc_minor, ret); 193 } else { 194 px_sdio_err_users++; 195 DBG(DBG_ATTACH, dip, "px_lib_dev_init: negotiated SDIO ERR API " 196 "version, major 0x%lx minor 0x%lx\n", 197 px_hsvc_sdio_err.hsvc_major, px_sdio_err_min_ver); 198 } 199 200 /* 201 * Find out the number of dev we need to wait under this RC 202 * before we issue fabric sync hypercall 203 */ 204 px_p->px_plat_p = (void *)(uintptr_t)px_lib_count_waiting_dev(dip); 205 DBG(DBG_ATTACH, dip, "Found %d bridges need waiting under RC %p", 206 (int)(uintptr_t)px_p->px_plat_p, dip); 207 return (DDI_SUCCESS); 208 } 209 210 /*ARGSUSED*/ 211 int 212 px_lib_dev_fini(dev_info_t *dip) 213 { 214 DBG(DBG_DETACH, dip, "px_lib_dev_fini: dip 0x%p\n", dip); 215 216 (void) ddi_prop_remove(makedevice(ddi_driver_major(dip), 217 PCI_MINOR_NUM(ddi_get_instance(dip), PCI_DEVCTL_MINOR)), dip, 218 PCI_BUS_CONF_MAP_PROP); 219 220 if (--px_vpci_users == 0) 221 (void) hsvc_unregister(&px_hsvc_vpci); 222 223 if (--px_sdio_users == 0) 224 (void) hsvc_unregister(&px_hsvc_sdio); 225 226 if (--px_sdio_err_users == 0) 227 (void) hsvc_unregister(&px_hsvc_sdio_err); 228 229 return (DDI_SUCCESS); 230 } 231 232 /*ARGSUSED*/ 233 int 234 px_lib_intr_devino_to_sysino(dev_info_t *dip, devino_t devino, 235 sysino_t *sysino) 236 { 237 uint64_t ret; 238 239 DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: dip 0x%p " 240 "devino 0x%x\n", dip, devino); 241 242 if ((ret = hvio_intr_devino_to_sysino(DIP_TO_HANDLE(dip), 243 devino, sysino)) != H_EOK) { 244 DBG(DBG_LIB_INT, dip, 245 "hvio_intr_devino_to_sysino failed, ret 0x%lx\n", ret); 246 return (DDI_FAILURE); 247 } 248 249 DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: sysino 0x%llx\n", 250 *sysino); 251 252 return (DDI_SUCCESS); 253 } 254 255 /*ARGSUSED*/ 256 int 257 px_lib_intr_getvalid(dev_info_t *dip, sysino_t sysino, 258 intr_valid_state_t *intr_valid_state) 259 { 260 uint64_t ret; 261 262 DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: dip 0x%p sysino 0x%llx\n", 263 dip, sysino); 264 265 if ((ret = hvio_intr_getvalid(sysino, 266 (int *)intr_valid_state)) != H_EOK) { 267 DBG(DBG_LIB_INT, dip, "hvio_intr_getvalid failed, ret 0x%lx\n", 268 ret); 269 return (DDI_FAILURE); 270 } 271 272 DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: intr_valid_state 0x%x\n", 273 *intr_valid_state); 274 275 return (DDI_SUCCESS); 276 } 277 278 /*ARGSUSED*/ 279 int 280 px_lib_intr_setvalid(dev_info_t *dip, sysino_t sysino, 281 intr_valid_state_t intr_valid_state) 282 { 283 uint64_t ret; 284 285 DBG(DBG_LIB_INT, dip, "px_lib_intr_setvalid: dip 0x%p sysino 0x%llx " 286 "intr_valid_state 0x%x\n", dip, sysino, intr_valid_state); 287 288 if ((ret = hvio_intr_setvalid(sysino, intr_valid_state)) != H_EOK) { 289 DBG(DBG_LIB_INT, dip, "hvio_intr_setvalid failed, ret 0x%lx\n", 290 ret); 291 return (DDI_FAILURE); 292 } 293 294 return (DDI_SUCCESS); 295 } 296 297 /*ARGSUSED*/ 298 int 299 px_lib_intr_getstate(dev_info_t *dip, sysino_t sysino, 300 intr_state_t *intr_state) 301 { 302 uint64_t ret; 303 304 DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: dip 0x%p sysino 0x%llx\n", 305 dip, sysino); 306 307 if ((ret = hvio_intr_getstate(sysino, (int *)intr_state)) != H_EOK) { 308 DBG(DBG_LIB_INT, dip, "hvio_intr_getstate failed, ret 0x%lx\n", 309 ret); 310 return (DDI_FAILURE); 311 } 312 313 DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: intr_state 0x%x\n", 314 *intr_state); 315 316 return (DDI_SUCCESS); 317 } 318 319 /*ARGSUSED*/ 320 int 321 px_lib_intr_setstate(dev_info_t *dip, sysino_t sysino, 322 intr_state_t intr_state) 323 { 324 uint64_t ret; 325 326 DBG(DBG_LIB_INT, dip, "px_lib_intr_setstate: dip 0x%p sysino 0x%llx " 327 "intr_state 0x%x\n", dip, sysino, intr_state); 328 329 if ((ret = hvio_intr_setstate(sysino, intr_state)) != H_EOK) { 330 DBG(DBG_LIB_INT, dip, "hvio_intr_setstate failed, ret 0x%lx\n", 331 ret); 332 return (DDI_FAILURE); 333 } 334 335 return (DDI_SUCCESS); 336 } 337 338 /*ARGSUSED*/ 339 int 340 px_lib_intr_gettarget(dev_info_t *dip, sysino_t sysino, cpuid_t *cpuid) 341 { 342 uint64_t ret; 343 344 DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: dip 0x%p sysino 0x%llx\n", 345 dip, sysino); 346 347 if ((ret = hvio_intr_gettarget(sysino, cpuid)) != H_EOK) { 348 DBG(DBG_LIB_INT, dip, 349 "hvio_intr_gettarget failed, ret 0x%lx\n", ret); 350 return (DDI_FAILURE); 351 } 352 353 DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: cpuid 0x%x\n", *cpuid); 354 355 return (DDI_SUCCESS); 356 } 357 358 /*ARGSUSED*/ 359 int 360 px_lib_intr_settarget(dev_info_t *dip, sysino_t sysino, cpuid_t cpuid) 361 { 362 uint64_t ret; 363 364 DBG(DBG_LIB_INT, dip, "px_lib_intr_settarget: dip 0x%p sysino 0x%llx " 365 "cpuid 0x%x\n", dip, sysino, cpuid); 366 367 ret = hvio_intr_settarget(sysino, cpuid); 368 if (ret == H_ECPUERROR) { 369 cmn_err(CE_PANIC, 370 "px_lib_intr_settarget: hvio_intr_settarget failed, " 371 "ret = 0x%lx, cpuid = 0x%x, sysino = 0x%lx\n", ret, 372 cpuid, sysino); 373 } else if (ret != H_EOK) { 374 DBG(DBG_LIB_INT, dip, 375 "hvio_intr_settarget failed, ret 0x%lx\n", ret); 376 return (DDI_FAILURE); 377 } 378 379 return (DDI_SUCCESS); 380 } 381 382 /*ARGSUSED*/ 383 int 384 px_lib_intr_reset(dev_info_t *dip) 385 { 386 px_t *px_p = DIP_TO_STATE(dip); 387 px_ib_t *ib_p = px_p->px_ib_p; 388 px_ino_t *ino_p; 389 390 DBG(DBG_LIB_INT, dip, "px_lib_intr_reset: dip 0x%p\n", dip); 391 392 mutex_enter(&ib_p->ib_ino_lst_mutex); 393 394 /* Reset all Interrupts */ 395 for (ino_p = ib_p->ib_ino_lst; ino_p; ino_p = ino_p->ino_next_p) { 396 if (px_lib_intr_setstate(dip, ino_p->ino_sysino, 397 INTR_IDLE_STATE) != DDI_SUCCESS) 398 return (BF_FATAL); 399 } 400 401 mutex_exit(&ib_p->ib_ino_lst_mutex); 402 403 return (BF_NONE); 404 } 405 406 /*ARGSUSED*/ 407 int 408 px_lib_iommu_map(dev_info_t *dip, tsbid_t tsbid, pages_t pages, 409 io_attributes_t attr, void *addr, size_t pfn_index, int flags) 410 { 411 tsbnum_t tsb_num = PCI_TSBID_TO_TSBNUM(tsbid); 412 tsbindex_t tsb_index = PCI_TSBID_TO_TSBINDEX(tsbid); 413 io_page_list_t *pfns, *pfn_p; 414 pages_t ttes_mapped = 0; 415 int i, err = DDI_SUCCESS; 416 417 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: dip 0x%p tsbid 0x%llx " 418 "pages 0x%x attr 0x%llx addr 0x%p pfn_index 0x%llx flags 0x%x\n", 419 dip, tsbid, pages, attr, addr, pfn_index, flags); 420 421 if ((pfns = pfn_p = kmem_zalloc((pages * sizeof (io_page_list_t)), 422 KM_NOSLEEP)) == NULL) { 423 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: kmem_zalloc failed\n"); 424 return (DDI_FAILURE); 425 } 426 427 for (i = 0; i < pages; i++) 428 pfns[i] = MMU_PTOB(PX_ADDR2PFN(addr, pfn_index, flags, i)); 429 430 /* 431 * If HV VPCI version is 1.1 and higher, pass BDF, phantom function, 432 * and relaxed ordering attributes. Otherwise, pass only read or write 433 * attribute. 434 */ 435 if (px_vpci_min_ver == PX_HSVC_MINOR_VER_0) 436 attr = attr & (PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE); 437 438 while ((ttes_mapped = pfn_p - pfns) < pages) { 439 uintptr_t ra = va_to_pa(pfn_p); 440 pages_t ttes2map; 441 uint64_t ret; 442 443 ttes2map = (MMU_PAGE_SIZE - P2PHASE(ra, MMU_PAGE_SIZE)) >> 3; 444 ra = MMU_PTOB(MMU_BTOP(ra)); 445 446 for (ttes2map = MIN(ttes2map, pages - ttes_mapped); ttes2map; 447 ttes2map -= ttes_mapped, pfn_p += ttes_mapped) { 448 449 ttes_mapped = 0; 450 if ((ret = hvio_iommu_map(DIP_TO_HANDLE(dip), 451 PCI_TSBID(tsb_num, tsb_index + (pfn_p - pfns)), 452 ttes2map, attr, (io_page_list_t *)(ra | 453 ((uintptr_t)pfn_p & MMU_PAGE_OFFSET)), 454 &ttes_mapped)) != H_EOK) { 455 DBG(DBG_LIB_DMA, dip, "hvio_iommu_map failed " 456 "ret 0x%lx\n", ret); 457 458 ttes_mapped = pfn_p - pfns; 459 err = DDI_FAILURE; 460 goto cleanup; 461 } 462 463 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: tsb_num 0x%x " 464 "tsb_index 0x%lx ttes_to_map 0x%lx attr 0x%llx " 465 "ra 0x%p ttes_mapped 0x%x\n", tsb_num, 466 tsb_index + (pfn_p - pfns), ttes2map, attr, 467 ra | ((uintptr_t)pfn_p & MMU_PAGE_OFFSET), 468 ttes_mapped); 469 } 470 } 471 472 cleanup: 473 if ((err == DDI_FAILURE) && ttes_mapped) 474 (void) px_lib_iommu_demap(dip, tsbid, ttes_mapped); 475 476 kmem_free(pfns, pages * sizeof (io_page_list_t)); 477 return (err); 478 } 479 480 /*ARGSUSED*/ 481 int 482 px_lib_iommu_demap(dev_info_t *dip, tsbid_t tsbid, pages_t pages) 483 { 484 tsbnum_t tsb_num = PCI_TSBID_TO_TSBNUM(tsbid); 485 tsbindex_t tsb_index = PCI_TSBID_TO_TSBINDEX(tsbid); 486 pages_t ttes2demap, ttes_demapped = 0; 487 uint64_t ret; 488 489 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: dip 0x%p tsbid 0x%llx " 490 "pages 0x%x\n", dip, tsbid, pages); 491 492 for (ttes2demap = pages; ttes2demap; 493 ttes2demap -= ttes_demapped, tsb_index += ttes_demapped) { 494 if ((ret = hvio_iommu_demap(DIP_TO_HANDLE(dip), 495 PCI_TSBID(tsb_num, tsb_index), ttes2demap, 496 &ttes_demapped)) != H_EOK) { 497 DBG(DBG_LIB_DMA, dip, "hvio_iommu_demap failed, " 498 "ret 0x%lx\n", ret); 499 500 return (DDI_FAILURE); 501 } 502 503 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: tsb_num 0x%x " 504 "tsb_index 0x%lx ttes_to_demap 0x%lx ttes_demapped 0x%x\n", 505 tsb_num, tsb_index, ttes2demap, ttes_demapped); 506 } 507 508 return (DDI_SUCCESS); 509 } 510 511 /*ARGSUSED*/ 512 int 513 px_lib_iommu_getmap(dev_info_t *dip, tsbid_t tsbid, io_attributes_t *attr_p, 514 r_addr_t *r_addr_p) 515 { 516 uint64_t ret; 517 518 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: dip 0x%p tsbid 0x%llx\n", 519 dip, tsbid); 520 521 if ((ret = hvio_iommu_getmap(DIP_TO_HANDLE(dip), tsbid, 522 attr_p, r_addr_p)) != H_EOK) { 523 DBG(DBG_LIB_DMA, dip, 524 "hvio_iommu_getmap failed, ret 0x%lx\n", ret); 525 526 return ((ret == H_ENOMAP) ? DDI_DMA_NOMAPPING:DDI_FAILURE); 527 } 528 529 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: attr 0x%llx " 530 "r_addr 0x%llx\n", *attr_p, *r_addr_p); 531 532 return (DDI_SUCCESS); 533 } 534 535 /*ARGSUSED*/ 536 uint64_t 537 px_get_rng_parent_hi_mask(px_t *px_p) 538 { 539 return (PX_RANGE_PROP_MASK); 540 } 541 542 /* 543 * Checks dma attributes against system bypass ranges 544 * A sun4v device must be capable of generating the entire 64-bit 545 * address in order to perform bypass DMA. 546 */ 547 /*ARGSUSED*/ 548 int 549 px_lib_dma_bypass_rngchk(dev_info_t *dip, ddi_dma_attr_t *attr_p, 550 uint64_t *lo_p, uint64_t *hi_p) 551 { 552 if ((attr_p->dma_attr_addr_lo != 0ull) || 553 (attr_p->dma_attr_addr_hi != UINT64_MAX)) { 554 555 return (DDI_DMA_BADATTR); 556 } 557 558 *lo_p = 0ull; 559 *hi_p = UINT64_MAX; 560 561 return (DDI_SUCCESS); 562 } 563 564 565 /*ARGSUSED*/ 566 int 567 px_lib_iommu_getbypass(dev_info_t *dip, r_addr_t ra, io_attributes_t attr, 568 io_addr_t *io_addr_p) 569 { 570 uint64_t ret; 571 572 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: dip 0x%p ra 0x%llx " 573 "attr 0x%llx\n", dip, ra, attr); 574 /* 575 * If HV VPCI version is 1.1 and higher, pass BDF, phantom function, 576 * and relaxed ordering attributes. Otherwise, pass only read or write 577 * attribute. 578 */ 579 if (px_vpci_min_ver == PX_HSVC_MINOR_VER_0) 580 attr &= PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE; 581 582 if ((ret = hvio_iommu_getbypass(DIP_TO_HANDLE(dip), ra, 583 attr, io_addr_p)) != H_EOK) { 584 DBG(DBG_LIB_DMA, dip, 585 "hvio_iommu_getbypass failed, ret 0x%lx\n", ret); 586 return (ret == H_ENOTSUPPORTED ? DDI_ENOTSUP : DDI_FAILURE); 587 } 588 589 DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: io_addr 0x%llx\n", 590 *io_addr_p); 591 592 return (DDI_SUCCESS); 593 } 594 595 /* 596 * Returns any needed IO address bit(s) for relaxed ordering in IOMMU 597 * bypass mode. 598 */ 599 /* ARGSUSED */ 600 uint64_t 601 px_lib_ro_bypass(dev_info_t *dip, io_attributes_t attr, uint64_t ioaddr) 602 { 603 return (ioaddr); 604 } 605 606 /*ARGSUSED*/ 607 int 608 px_lib_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 609 off_t off, size_t len, uint_t cache_flags) 610 { 611 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle; 612 uint64_t sync_dir; 613 size_t bytes_synced; 614 int end, idx; 615 off_t pg_off; 616 devhandle_t hdl = DIP_TO_HANDLE(dip); /* need to cache hdl */ 617 618 DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: dip 0x%p rdip 0x%p " 619 "handle 0x%llx off 0x%x len 0x%x flags 0x%x\n", 620 dip, rdip, handle, off, len, cache_flags); 621 622 if (!(mp->dmai_flags & PX_DMAI_FLAGS_INUSE)) { 623 cmn_err(CE_WARN, "%s%d: Unbound dma handle %p.", 624 ddi_driver_name(rdip), ddi_get_instance(rdip), (void *)mp); 625 return (DDI_FAILURE); 626 } 627 628 if (mp->dmai_flags & PX_DMAI_FLAGS_NOSYNC) 629 return (DDI_SUCCESS); 630 631 if (!len) 632 len = mp->dmai_size; 633 634 if (mp->dmai_rflags & DDI_DMA_READ) 635 sync_dir = HVIO_DMA_SYNC_DIR_FROM_DEV; 636 else 637 sync_dir = HVIO_DMA_SYNC_DIR_TO_DEV; 638 639 off += mp->dmai_offset; 640 pg_off = off & MMU_PAGEOFFSET; 641 642 DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: page offset %x size %x\n", 643 pg_off, len); 644 645 /* sync on page basis */ 646 end = MMU_BTOPR(off + len - 1); 647 for (idx = MMU_BTOP(off); idx < end; idx++, 648 len -= bytes_synced, pg_off = 0) { 649 size_t bytes_to_sync = bytes_to_sync = 650 MIN(len, MMU_PAGESIZE - pg_off); 651 652 if (hvio_dma_sync(hdl, MMU_PTOB(PX_GET_MP_PFN(mp, idx)) + 653 pg_off, bytes_to_sync, sync_dir, &bytes_synced) != H_EOK) 654 break; 655 656 DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: Called hvio_dma_sync " 657 "ra = %p bytes to sync = %x bytes synced %x\n", 658 MMU_PTOB(PX_GET_MP_PFN(mp, idx)) + pg_off, bytes_to_sync, 659 bytes_synced); 660 661 if (bytes_to_sync != bytes_synced) 662 break; 663 } 664 665 return (len ? DDI_FAILURE : DDI_SUCCESS); 666 } 667 668 669 /* 670 * MSIQ Functions: 671 */ 672 673 /*ARGSUSED*/ 674 int 675 px_lib_msiq_init(dev_info_t *dip) 676 { 677 px_t *px_p = DIP_TO_STATE(dip); 678 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state; 679 r_addr_t ra; 680 size_t msiq_size; 681 uint_t rec_cnt; 682 int i, err = DDI_SUCCESS; 683 uint64_t ret; 684 685 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_init: dip 0x%p\n", dip); 686 687 msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t); 688 689 /* sun4v requires all EQ allocation to be on q size boundary */ 690 if ((msiq_state_p->msiq_buf_p = contig_mem_alloc_align( 691 msiq_state_p->msiq_cnt * msiq_size, msiq_size)) == NULL) { 692 DBG(DBG_LIB_MSIQ, dip, 693 "px_lib_msiq_init: Contig alloc failed\n"); 694 695 return (DDI_FAILURE); 696 } 697 698 for (i = 0; i < msiq_state_p->msiq_cnt; i++) { 699 msiq_state_p->msiq_p[i].msiq_base_p = (msiqhead_t *) 700 ((caddr_t)msiq_state_p->msiq_buf_p + (i * msiq_size)); 701 702 ra = (r_addr_t)va_to_pa((caddr_t)msiq_state_p->msiq_buf_p + 703 (i * msiq_size)); 704 705 if ((ret = hvio_msiq_conf(DIP_TO_HANDLE(dip), 706 (i + msiq_state_p->msiq_1st_msiq_id), 707 ra, msiq_state_p->msiq_rec_cnt)) != H_EOK) { 708 DBG(DBG_LIB_MSIQ, dip, 709 "hvio_msiq_conf failed, ret 0x%lx\n", ret); 710 err = DDI_FAILURE; 711 break; 712 } 713 714 if ((err = px_lib_msiq_info(dip, 715 (i + msiq_state_p->msiq_1st_msiq_id), 716 &ra, &rec_cnt)) != DDI_SUCCESS) { 717 DBG(DBG_LIB_MSIQ, dip, 718 "px_lib_msiq_info failed, ret 0x%x\n", err); 719 err = DDI_FAILURE; 720 break; 721 } 722 723 DBG(DBG_LIB_MSIQ, dip, 724 "px_lib_msiq_init: ra 0x%p rec_cnt 0x%x\n", ra, rec_cnt); 725 } 726 727 return (err); 728 } 729 730 /*ARGSUSED*/ 731 int 732 px_lib_msiq_fini(dev_info_t *dip) 733 { 734 px_t *px_p = DIP_TO_STATE(dip); 735 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state; 736 size_t msiq_size; 737 738 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_fini: dip 0x%p\n", dip); 739 msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t); 740 741 if (msiq_state_p->msiq_buf_p != NULL) 742 contig_mem_free(msiq_state_p->msiq_buf_p, 743 msiq_state_p->msiq_cnt * msiq_size); 744 745 return (DDI_SUCCESS); 746 } 747 748 /*ARGSUSED*/ 749 int 750 px_lib_msiq_info(dev_info_t *dip, msiqid_t msiq_id, r_addr_t *ra_p, 751 uint_t *msiq_rec_cnt_p) 752 { 753 uint64_t ret; 754 755 DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: dip 0x%p msiq_id 0x%x\n", 756 dip, msiq_id); 757 758 if ((ret = hvio_msiq_info(DIP_TO_HANDLE(dip), 759 msiq_id, ra_p, msiq_rec_cnt_p)) != H_EOK) { 760 DBG(DBG_LIB_MSIQ, dip, 761 "hvio_msiq_info failed, ret 0x%lx\n", ret); 762 return (DDI_FAILURE); 763 } 764 765 DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: ra_p 0x%p msiq_rec_cnt 0x%x\n", 766 ra_p, *msiq_rec_cnt_p); 767 768 return (DDI_SUCCESS); 769 } 770 771 /*ARGSUSED*/ 772 int 773 px_lib_msiq_getvalid(dev_info_t *dip, msiqid_t msiq_id, 774 pci_msiq_valid_state_t *msiq_valid_state) 775 { 776 uint64_t ret; 777 778 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: dip 0x%p msiq_id 0x%x\n", 779 dip, msiq_id); 780 781 if ((ret = hvio_msiq_getvalid(DIP_TO_HANDLE(dip), 782 msiq_id, msiq_valid_state)) != H_EOK) { 783 DBG(DBG_LIB_MSIQ, dip, 784 "hvio_msiq_getvalid failed, ret 0x%lx\n", ret); 785 return (DDI_FAILURE); 786 } 787 788 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: msiq_valid_state 0x%x\n", 789 *msiq_valid_state); 790 791 return (DDI_SUCCESS); 792 } 793 794 /*ARGSUSED*/ 795 int 796 px_lib_msiq_setvalid(dev_info_t *dip, msiqid_t msiq_id, 797 pci_msiq_valid_state_t msiq_valid_state) 798 { 799 uint64_t ret; 800 801 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setvalid: dip 0x%p msiq_id 0x%x " 802 "msiq_valid_state 0x%x\n", dip, msiq_id, msiq_valid_state); 803 804 if ((ret = hvio_msiq_setvalid(DIP_TO_HANDLE(dip), 805 msiq_id, msiq_valid_state)) != H_EOK) { 806 DBG(DBG_LIB_MSIQ, dip, 807 "hvio_msiq_setvalid failed, ret 0x%lx\n", ret); 808 return (DDI_FAILURE); 809 } 810 811 return (DDI_SUCCESS); 812 } 813 814 /*ARGSUSED*/ 815 int 816 px_lib_msiq_getstate(dev_info_t *dip, msiqid_t msiq_id, 817 pci_msiq_state_t *msiq_state) 818 { 819 uint64_t ret; 820 821 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: dip 0x%p msiq_id 0x%x\n", 822 dip, msiq_id); 823 824 if ((ret = hvio_msiq_getstate(DIP_TO_HANDLE(dip), 825 msiq_id, msiq_state)) != H_EOK) { 826 DBG(DBG_LIB_MSIQ, dip, 827 "hvio_msiq_getstate failed, ret 0x%lx\n", ret); 828 return (DDI_FAILURE); 829 } 830 831 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: msiq_state 0x%x\n", 832 *msiq_state); 833 834 return (DDI_SUCCESS); 835 } 836 837 /*ARGSUSED*/ 838 int 839 px_lib_msiq_setstate(dev_info_t *dip, msiqid_t msiq_id, 840 pci_msiq_state_t msiq_state) 841 { 842 uint64_t ret; 843 844 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setstate: dip 0x%p msiq_id 0x%x " 845 "msiq_state 0x%x\n", dip, msiq_id, msiq_state); 846 847 if ((ret = hvio_msiq_setstate(DIP_TO_HANDLE(dip), 848 msiq_id, msiq_state)) != H_EOK) { 849 DBG(DBG_LIB_MSIQ, dip, 850 "hvio_msiq_setstate failed, ret 0x%lx\n", ret); 851 return (DDI_FAILURE); 852 } 853 854 return (DDI_SUCCESS); 855 } 856 857 /*ARGSUSED*/ 858 int 859 px_lib_msiq_gethead(dev_info_t *dip, msiqid_t msiq_id, 860 msiqhead_t *msiq_head_p) 861 { 862 uint64_t ret; 863 864 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: dip 0x%p msiq_id 0x%x\n", 865 dip, msiq_id); 866 867 if ((ret = hvio_msiq_gethead(DIP_TO_HANDLE(dip), 868 msiq_id, msiq_head_p)) != H_EOK) { 869 DBG(DBG_LIB_MSIQ, dip, 870 "hvio_msiq_gethead failed, ret 0x%lx\n", ret); 871 return (DDI_FAILURE); 872 } 873 874 *msiq_head_p = (*msiq_head_p / sizeof (msiq_rec_t)); 875 876 DBG(DBG_LIB_MSIQ, dip, "px_msiq_gethead: msiq_head 0x%x\n", 877 *msiq_head_p); 878 879 return (DDI_SUCCESS); 880 } 881 882 /*ARGSUSED*/ 883 int 884 px_lib_msiq_sethead(dev_info_t *dip, msiqid_t msiq_id, 885 msiqhead_t msiq_head) 886 { 887 uint64_t ret; 888 889 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_sethead: dip 0x%p msiq_id 0x%x " 890 "msiq_head 0x%x\n", dip, msiq_id, msiq_head); 891 892 if ((ret = hvio_msiq_sethead(DIP_TO_HANDLE(dip), 893 msiq_id, msiq_head * sizeof (msiq_rec_t))) != H_EOK) { 894 DBG(DBG_LIB_MSIQ, dip, 895 "hvio_msiq_sethead failed, ret 0x%lx\n", ret); 896 return (DDI_FAILURE); 897 } 898 899 return (DDI_SUCCESS); 900 } 901 902 /*ARGSUSED*/ 903 int 904 px_lib_msiq_gettail(dev_info_t *dip, msiqid_t msiq_id, 905 msiqtail_t *msiq_tail_p) 906 { 907 uint64_t ret; 908 909 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: dip 0x%p msiq_id 0x%x\n", 910 dip, msiq_id); 911 912 if ((ret = hvio_msiq_gettail(DIP_TO_HANDLE(dip), 913 msiq_id, msiq_tail_p)) != H_EOK) { 914 DBG(DBG_LIB_MSIQ, dip, 915 "hvio_msiq_gettail failed, ret 0x%lx\n", ret); 916 return (DDI_FAILURE); 917 } 918 919 *msiq_tail_p = (*msiq_tail_p / sizeof (msiq_rec_t)); 920 DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: msiq_tail 0x%x\n", 921 *msiq_tail_p); 922 923 return (DDI_SUCCESS); 924 } 925 926 /*ARGSUSED*/ 927 void 928 px_lib_get_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p, 929 msiq_rec_t *msiq_rec_p) 930 { 931 msiq_rec_t *curr_msiq_rec_p = (msiq_rec_t *)msiq_head_p; 932 933 DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: dip 0x%p\n", dip); 934 935 if (!curr_msiq_rec_p->msiq_rec_type) { 936 /* Set msiq_rec_type to zero */ 937 msiq_rec_p->msiq_rec_type = 0; 938 939 return; 940 } 941 942 *msiq_rec_p = *curr_msiq_rec_p; 943 } 944 945 /*ARGSUSED*/ 946 void 947 px_lib_clr_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p) 948 { 949 msiq_rec_t *curr_msiq_rec_p = (msiq_rec_t *)msiq_head_p; 950 951 DBG(DBG_LIB_MSIQ, dip, "px_lib_clr_msiq_rec: dip 0x%p\n", dip); 952 953 /* Zero out msiq_rec_type field */ 954 curr_msiq_rec_p->msiq_rec_type = 0; 955 } 956 957 /* 958 * MSI Functions: 959 */ 960 961 /*ARGSUSED*/ 962 int 963 px_lib_msi_init(dev_info_t *dip) 964 { 965 DBG(DBG_LIB_MSI, dip, "px_lib_msi_init: dip 0x%p\n", dip); 966 967 /* Noop */ 968 return (DDI_SUCCESS); 969 } 970 971 /*ARGSUSED*/ 972 int 973 px_lib_msi_getmsiq(dev_info_t *dip, msinum_t msi_num, 974 msiqid_t *msiq_id) 975 { 976 uint64_t ret; 977 978 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: dip 0x%p msi_num 0x%x\n", 979 dip, msi_num); 980 981 if ((ret = hvio_msi_getmsiq(DIP_TO_HANDLE(dip), 982 msi_num, msiq_id)) != H_EOK) { 983 DBG(DBG_LIB_MSI, dip, 984 "hvio_msi_getmsiq failed, ret 0x%lx\n", ret); 985 return (DDI_FAILURE); 986 } 987 988 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: msiq_id 0x%x\n", 989 *msiq_id); 990 991 return (DDI_SUCCESS); 992 } 993 994 /*ARGSUSED*/ 995 int 996 px_lib_msi_setmsiq(dev_info_t *dip, msinum_t msi_num, 997 msiqid_t msiq_id, msi_type_t msitype) 998 { 999 uint64_t ret; 1000 1001 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setmsiq: dip 0x%p msi_num 0x%x " 1002 "msq_id 0x%x\n", dip, msi_num, msiq_id); 1003 1004 if ((ret = hvio_msi_setmsiq(DIP_TO_HANDLE(dip), 1005 msi_num, msiq_id, msitype)) != H_EOK) { 1006 DBG(DBG_LIB_MSI, dip, 1007 "hvio_msi_setmsiq failed, ret 0x%lx\n", ret); 1008 return (DDI_FAILURE); 1009 } 1010 1011 return (DDI_SUCCESS); 1012 } 1013 1014 /*ARGSUSED*/ 1015 int 1016 px_lib_msi_getvalid(dev_info_t *dip, msinum_t msi_num, 1017 pci_msi_valid_state_t *msi_valid_state) 1018 { 1019 uint64_t ret; 1020 1021 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: dip 0x%p msi_num 0x%x\n", 1022 dip, msi_num); 1023 1024 if ((ret = hvio_msi_getvalid(DIP_TO_HANDLE(dip), 1025 msi_num, msi_valid_state)) != H_EOK) { 1026 DBG(DBG_LIB_MSI, dip, 1027 "hvio_msi_getvalid failed, ret 0x%lx\n", ret); 1028 return (DDI_FAILURE); 1029 } 1030 1031 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: msiq_id 0x%x\n", 1032 *msi_valid_state); 1033 1034 return (DDI_SUCCESS); 1035 } 1036 1037 /*ARGSUSED*/ 1038 int 1039 px_lib_msi_setvalid(dev_info_t *dip, msinum_t msi_num, 1040 pci_msi_valid_state_t msi_valid_state) 1041 { 1042 uint64_t ret; 1043 1044 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setvalid: dip 0x%p msi_num 0x%x " 1045 "msi_valid_state 0x%x\n", dip, msi_num, msi_valid_state); 1046 1047 if ((ret = hvio_msi_setvalid(DIP_TO_HANDLE(dip), 1048 msi_num, msi_valid_state)) != H_EOK) { 1049 DBG(DBG_LIB_MSI, dip, 1050 "hvio_msi_setvalid failed, ret 0x%lx\n", ret); 1051 return (DDI_FAILURE); 1052 } 1053 1054 return (DDI_SUCCESS); 1055 } 1056 1057 /*ARGSUSED*/ 1058 int 1059 px_lib_msi_getstate(dev_info_t *dip, msinum_t msi_num, 1060 pci_msi_state_t *msi_state) 1061 { 1062 uint64_t ret; 1063 1064 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: dip 0x%p msi_num 0x%x\n", 1065 dip, msi_num); 1066 1067 if ((ret = hvio_msi_getstate(DIP_TO_HANDLE(dip), 1068 msi_num, msi_state)) != H_EOK) { 1069 DBG(DBG_LIB_MSI, dip, 1070 "hvio_msi_getstate failed, ret 0x%lx\n", ret); 1071 return (DDI_FAILURE); 1072 } 1073 1074 DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: msi_state 0x%x\n", 1075 *msi_state); 1076 1077 return (DDI_SUCCESS); 1078 } 1079 1080 /*ARGSUSED*/ 1081 int 1082 px_lib_msi_setstate(dev_info_t *dip, msinum_t msi_num, 1083 pci_msi_state_t msi_state) 1084 { 1085 uint64_t ret; 1086 1087 DBG(DBG_LIB_MSI, dip, "px_lib_msi_setstate: dip 0x%p msi_num 0x%x " 1088 "msi_state 0x%x\n", dip, msi_num, msi_state); 1089 1090 if ((ret = hvio_msi_setstate(DIP_TO_HANDLE(dip), 1091 msi_num, msi_state)) != H_EOK) { 1092 DBG(DBG_LIB_MSI, dip, 1093 "hvio_msi_setstate failed, ret 0x%lx\n", ret); 1094 return (DDI_FAILURE); 1095 } 1096 1097 return (DDI_SUCCESS); 1098 } 1099 1100 /* 1101 * MSG Functions: 1102 */ 1103 1104 /*ARGSUSED*/ 1105 int 1106 px_lib_msg_getmsiq(dev_info_t *dip, pcie_msg_type_t msg_type, 1107 msiqid_t *msiq_id) 1108 { 1109 uint64_t ret; 1110 1111 DBG(DBG_LIB_MSG, dip, "px_lib_msg_getmsiq: dip 0x%p msg_type 0x%x\n", 1112 dip, msg_type); 1113 1114 if ((ret = hvio_msg_getmsiq(DIP_TO_HANDLE(dip), 1115 msg_type, msiq_id)) != H_EOK) { 1116 DBG(DBG_LIB_MSG, dip, 1117 "hvio_msg_getmsiq failed, ret 0x%lx\n", ret); 1118 return (DDI_FAILURE); 1119 } 1120 1121 DBG(DBG_LIB_MSI, dip, "px_lib_msg_getmsiq: msiq_id 0x%x\n", 1122 *msiq_id); 1123 1124 return (DDI_SUCCESS); 1125 } 1126 1127 /*ARGSUSED*/ 1128 int 1129 px_lib_msg_setmsiq(dev_info_t *dip, pcie_msg_type_t msg_type, 1130 msiqid_t msiq_id) 1131 { 1132 uint64_t ret; 1133 1134 DBG(DBG_LIB_MSG, dip, "px_lib_msg_setmsiq: dip 0x%p msg_type 0x%x " 1135 "msq_id 0x%x\n", dip, msg_type, msiq_id); 1136 1137 if ((ret = hvio_msg_setmsiq(DIP_TO_HANDLE(dip), 1138 msg_type, msiq_id)) != H_EOK) { 1139 DBG(DBG_LIB_MSG, dip, 1140 "hvio_msg_setmsiq failed, ret 0x%lx\n", ret); 1141 return (DDI_FAILURE); 1142 } 1143 1144 return (DDI_SUCCESS); 1145 } 1146 1147 /*ARGSUSED*/ 1148 int 1149 px_lib_msg_getvalid(dev_info_t *dip, pcie_msg_type_t msg_type, 1150 pcie_msg_valid_state_t *msg_valid_state) 1151 { 1152 uint64_t ret; 1153 1154 DBG(DBG_LIB_MSG, dip, "px_lib_msg_getvalid: dip 0x%p msg_type 0x%x\n", 1155 dip, msg_type); 1156 1157 if ((ret = hvio_msg_getvalid(DIP_TO_HANDLE(dip), msg_type, 1158 msg_valid_state)) != H_EOK) { 1159 DBG(DBG_LIB_MSG, dip, 1160 "hvio_msg_getvalid failed, ret 0x%lx\n", ret); 1161 return (DDI_FAILURE); 1162 } 1163 1164 DBG(DBG_LIB_MSI, dip, "px_lib_msg_getvalid: msg_valid_state 0x%x\n", 1165 *msg_valid_state); 1166 1167 return (DDI_SUCCESS); 1168 } 1169 1170 /*ARGSUSED*/ 1171 int 1172 px_lib_msg_setvalid(dev_info_t *dip, pcie_msg_type_t msg_type, 1173 pcie_msg_valid_state_t msg_valid_state) 1174 { 1175 uint64_t ret; 1176 1177 DBG(DBG_LIB_MSG, dip, "px_lib_msg_setvalid: dip 0x%p msg_type 0x%x " 1178 "msg_valid_state 0x%x\n", dip, msg_type, msg_valid_state); 1179 1180 if ((ret = hvio_msg_setvalid(DIP_TO_HANDLE(dip), msg_type, 1181 msg_valid_state)) != H_EOK) { 1182 DBG(DBG_LIB_MSG, dip, 1183 "hvio_msg_setvalid failed, ret 0x%lx\n", ret); 1184 return (DDI_FAILURE); 1185 } 1186 1187 return (DDI_SUCCESS); 1188 } 1189 1190 /* 1191 * Suspend/Resume Functions: 1192 * Currently unsupported by hypervisor and all functions are noops. 1193 */ 1194 /*ARGSUSED*/ 1195 int 1196 px_lib_suspend(dev_info_t *dip) 1197 { 1198 DBG(DBG_ATTACH, dip, "px_lib_suspend: Not supported\n"); 1199 1200 /* Not supported */ 1201 return (DDI_FAILURE); 1202 } 1203 1204 /*ARGSUSED*/ 1205 void 1206 px_lib_resume(dev_info_t *dip) 1207 { 1208 DBG(DBG_ATTACH, dip, "px_lib_resume: Not supported\n"); 1209 1210 /* Noop */ 1211 } 1212 1213 /* 1214 * Misc Functions: 1215 * Currently unsupported by hypervisor and all functions are noops. 1216 */ 1217 /*ARGSUSED*/ 1218 static int 1219 px_lib_config_get(dev_info_t *dip, pci_device_t bdf, pci_config_offset_t off, 1220 uint8_t size, pci_cfg_data_t *data_p) 1221 { 1222 uint64_t ret; 1223 1224 DBG(DBG_LIB_CFG, dip, "px_lib_config_get: dip 0x%p, bdf 0x%llx " 1225 "off 0x%x size 0x%x\n", dip, bdf, off, size); 1226 1227 if ((ret = hvio_config_get(DIP_TO_HANDLE(dip), bdf, off, 1228 size, data_p)) != H_EOK) { 1229 DBG(DBG_LIB_CFG, dip, 1230 "hvio_config_get failed, ret 0x%lx\n", ret); 1231 return (DDI_FAILURE); 1232 } 1233 DBG(DBG_LIB_CFG, dip, "px_config_get: data 0x%x\n", data_p->dw); 1234 1235 return (DDI_SUCCESS); 1236 } 1237 1238 /*ARGSUSED*/ 1239 static int 1240 px_lib_config_put(dev_info_t *dip, pci_device_t bdf, pci_config_offset_t off, 1241 uint8_t size, pci_cfg_data_t data) 1242 { 1243 uint64_t ret; 1244 1245 DBG(DBG_LIB_CFG, dip, "px_lib_config_put: dip 0x%p, bdf 0x%llx " 1246 "off 0x%x size 0x%x data 0x%llx\n", dip, bdf, off, size, data.qw); 1247 1248 if ((ret = hvio_config_put(DIP_TO_HANDLE(dip), bdf, off, 1249 size, data)) != H_EOK) { 1250 DBG(DBG_LIB_CFG, dip, 1251 "hvio_config_put failed, ret 0x%lx\n", ret); 1252 return (DDI_FAILURE); 1253 } 1254 1255 return (DDI_SUCCESS); 1256 } 1257 1258 static uint32_t 1259 px_pci_config_get(ddi_acc_impl_t *handle, uint32_t *addr, int size) 1260 { 1261 px_config_acc_pvt_t *px_pvt = (px_config_acc_pvt_t *) 1262 handle->ahi_common.ah_bus_private; 1263 pcie_bus_t *busp = NULL; 1264 dev_info_t *cdip = NULL; 1265 uint32_t pci_dev_addr = px_pvt->raddr; 1266 uint32_t vaddr = px_pvt->vaddr; 1267 uint16_t off = (uint16_t)(uintptr_t)(addr - vaddr) & 0xfff; 1268 uint64_t rdata = 0; 1269 1270 if (px_lib_config_get(px_pvt->dip, pci_dev_addr, off, 1271 size, (pci_cfg_data_t *)&rdata) != DDI_SUCCESS) 1272 /* XXX update error kstats */ 1273 return (0xffffffff); 1274 1275 if (cdip = pcie_find_dip_by_bdf(px_pvt->dip, pci_dev_addr >> 8)) 1276 busp = PCIE_DIP2BUS(cdip); 1277 /* 1278 * This can be called early, before busp or busp->bus_dom has 1279 * been initialized, so check both before invoking 1280 * PCIE_IS_ASSIGNED. 1281 */ 1282 if (busp && PCIE_BUS2DOM(busp) && PCIE_IS_ASSIGNED(busp)) { 1283 if (off == PCI_CONF_VENID && size == 2) 1284 rdata = busp->bus_dev_ven_id & 0xffff; 1285 else if (off == PCI_CONF_DEVID && size == 2) 1286 rdata = busp->bus_dev_ven_id >> 16; 1287 else if (off == PCI_CONF_VENID && size == 4) 1288 rdata = busp->bus_dev_ven_id; 1289 } 1290 return ((uint32_t)rdata); 1291 } 1292 1293 static void 1294 px_pci_config_put(ddi_acc_impl_t *handle, uint32_t *addr, 1295 int size, pci_cfg_data_t wdata) 1296 { 1297 px_config_acc_pvt_t *px_pvt = (px_config_acc_pvt_t *) 1298 handle->ahi_common.ah_bus_private; 1299 uint32_t pci_dev_addr = px_pvt->raddr; 1300 uint32_t vaddr = px_pvt->vaddr; 1301 uint16_t off = (uint16_t)(uintptr_t)(addr - vaddr) & 0xfff; 1302 1303 if (px_lib_config_put(px_pvt->dip, pci_dev_addr, off, 1304 size, wdata) != DDI_SUCCESS) { 1305 /*EMPTY*/ 1306 /* XXX update error kstats */ 1307 } 1308 } 1309 1310 static uint8_t 1311 px_pci_config_get8(ddi_acc_impl_t *handle, uint8_t *addr) 1312 { 1313 return ((uint8_t)px_pci_config_get(handle, (uint32_t *)addr, 1)); 1314 } 1315 1316 static uint16_t 1317 px_pci_config_get16(ddi_acc_impl_t *handle, uint16_t *addr) 1318 { 1319 return ((uint16_t)px_pci_config_get(handle, (uint32_t *)addr, 2)); 1320 } 1321 1322 static uint32_t 1323 px_pci_config_get32(ddi_acc_impl_t *handle, uint32_t *addr) 1324 { 1325 return ((uint32_t)px_pci_config_get(handle, (uint32_t *)addr, 4)); 1326 } 1327 1328 static uint64_t 1329 px_pci_config_get64(ddi_acc_impl_t *handle, uint64_t *addr) 1330 { 1331 uint32_t rdatah, rdatal; 1332 1333 rdatal = (uint32_t)px_pci_config_get(handle, (uint32_t *)addr, 4); 1334 rdatah = (uint32_t)px_pci_config_get(handle, 1335 (uint32_t *)((char *)addr+4), 4); 1336 return (((uint64_t)rdatah << 32) | rdatal); 1337 } 1338 1339 static void 1340 px_pci_config_put8(ddi_acc_impl_t *handle, uint8_t *addr, uint8_t data) 1341 { 1342 pci_cfg_data_t wdata = { 0 }; 1343 1344 wdata.qw = (uint8_t)data; 1345 px_pci_config_put(handle, (uint32_t *)addr, 1, wdata); 1346 } 1347 1348 static void 1349 px_pci_config_put16(ddi_acc_impl_t *handle, uint16_t *addr, uint16_t data) 1350 { 1351 pci_cfg_data_t wdata = { 0 }; 1352 1353 wdata.qw = (uint16_t)data; 1354 px_pci_config_put(handle, (uint32_t *)addr, 2, wdata); 1355 } 1356 1357 static void 1358 px_pci_config_put32(ddi_acc_impl_t *handle, uint32_t *addr, uint32_t data) 1359 { 1360 pci_cfg_data_t wdata = { 0 }; 1361 1362 wdata.qw = (uint32_t)data; 1363 px_pci_config_put(handle, (uint32_t *)addr, 4, wdata); 1364 } 1365 1366 static void 1367 px_pci_config_put64(ddi_acc_impl_t *handle, uint64_t *addr, uint64_t data) 1368 { 1369 pci_cfg_data_t wdata = { 0 }; 1370 1371 wdata.qw = (uint32_t)(data & 0xffffffff); 1372 px_pci_config_put(handle, (uint32_t *)addr, 4, wdata); 1373 wdata.qw = (uint32_t)((data >> 32) & 0xffffffff); 1374 px_pci_config_put(handle, (uint32_t *)((char *)addr+4), 4, wdata); 1375 } 1376 1377 static void 1378 px_pci_config_rep_get8(ddi_acc_impl_t *handle, uint8_t *host_addr, 1379 uint8_t *dev_addr, size_t repcount, uint_t flags) 1380 { 1381 if (flags == DDI_DEV_AUTOINCR) 1382 for (; repcount; repcount--) 1383 *host_addr++ = px_pci_config_get8(handle, dev_addr++); 1384 else 1385 for (; repcount; repcount--) 1386 *host_addr++ = px_pci_config_get8(handle, dev_addr); 1387 } 1388 1389 /* 1390 * Function to rep read 16 bit data off the PCI configuration space behind 1391 * the 21554's host interface. 1392 */ 1393 static void 1394 px_pci_config_rep_get16(ddi_acc_impl_t *handle, uint16_t *host_addr, 1395 uint16_t *dev_addr, size_t repcount, uint_t flags) 1396 { 1397 if (flags == DDI_DEV_AUTOINCR) 1398 for (; repcount; repcount--) 1399 *host_addr++ = px_pci_config_get16(handle, dev_addr++); 1400 else 1401 for (; repcount; repcount--) 1402 *host_addr++ = px_pci_config_get16(handle, dev_addr); 1403 } 1404 1405 /* 1406 * Function to rep read 32 bit data off the PCI configuration space behind 1407 * the 21554's host interface. 1408 */ 1409 static void 1410 px_pci_config_rep_get32(ddi_acc_impl_t *handle, uint32_t *host_addr, 1411 uint32_t *dev_addr, size_t repcount, uint_t flags) 1412 { 1413 if (flags == DDI_DEV_AUTOINCR) 1414 for (; repcount; repcount--) 1415 *host_addr++ = px_pci_config_get32(handle, dev_addr++); 1416 else 1417 for (; repcount; repcount--) 1418 *host_addr++ = px_pci_config_get32(handle, dev_addr); 1419 } 1420 1421 /* 1422 * Function to rep read 64 bit data off the PCI configuration space behind 1423 * the 21554's host interface. 1424 */ 1425 static void 1426 px_pci_config_rep_get64(ddi_acc_impl_t *handle, uint64_t *host_addr, 1427 uint64_t *dev_addr, size_t repcount, uint_t flags) 1428 { 1429 if (flags == DDI_DEV_AUTOINCR) 1430 for (; repcount; repcount--) 1431 *host_addr++ = px_pci_config_get64(handle, dev_addr++); 1432 else 1433 for (; repcount; repcount--) 1434 *host_addr++ = px_pci_config_get64(handle, dev_addr); 1435 } 1436 1437 /* 1438 * Function to rep write 8 bit data into the PCI configuration space behind 1439 * the 21554's host interface. 1440 */ 1441 static void 1442 px_pci_config_rep_put8(ddi_acc_impl_t *handle, uint8_t *host_addr, 1443 uint8_t *dev_addr, size_t repcount, uint_t flags) 1444 { 1445 if (flags == DDI_DEV_AUTOINCR) 1446 for (; repcount; repcount--) 1447 px_pci_config_put8(handle, dev_addr++, *host_addr++); 1448 else 1449 for (; repcount; repcount--) 1450 px_pci_config_put8(handle, dev_addr, *host_addr++); 1451 } 1452 1453 /* 1454 * Function to rep write 16 bit data into the PCI configuration space behind 1455 * the 21554's host interface. 1456 */ 1457 static void 1458 px_pci_config_rep_put16(ddi_acc_impl_t *handle, uint16_t *host_addr, 1459 uint16_t *dev_addr, size_t repcount, uint_t flags) 1460 { 1461 if (flags == DDI_DEV_AUTOINCR) 1462 for (; repcount; repcount--) 1463 px_pci_config_put16(handle, dev_addr++, *host_addr++); 1464 else 1465 for (; repcount; repcount--) 1466 px_pci_config_put16(handle, dev_addr, *host_addr++); 1467 } 1468 1469 /* 1470 * Function to rep write 32 bit data into the PCI configuration space behind 1471 * the 21554's host interface. 1472 */ 1473 static void 1474 px_pci_config_rep_put32(ddi_acc_impl_t *handle, uint32_t *host_addr, 1475 uint32_t *dev_addr, size_t repcount, uint_t flags) 1476 { 1477 if (flags == DDI_DEV_AUTOINCR) 1478 for (; repcount; repcount--) 1479 px_pci_config_put32(handle, dev_addr++, *host_addr++); 1480 else 1481 for (; repcount; repcount--) 1482 px_pci_config_put32(handle, dev_addr, *host_addr++); 1483 } 1484 1485 /* 1486 * Function to rep write 64 bit data into the PCI configuration space behind 1487 * the 21554's host interface. 1488 */ 1489 static void 1490 px_pci_config_rep_put64(ddi_acc_impl_t *handle, uint64_t *host_addr, 1491 uint64_t *dev_addr, size_t repcount, uint_t flags) 1492 { 1493 if (flags == DDI_DEV_AUTOINCR) 1494 for (; repcount; repcount--) 1495 px_pci_config_put64(handle, dev_addr++, *host_addr++); 1496 else 1497 for (; repcount; repcount--) 1498 px_pci_config_put64(handle, dev_addr, *host_addr++); 1499 } 1500 1501 /* 1502 * Provide a private access handle to route config access calls to Hypervisor. 1503 * Beware: Do all error checking for config space accesses before calling 1504 * this function. ie. do error checking from the calling function. 1505 * Due to a lack of meaningful error code in DDI, the gauranteed return of 1506 * DDI_SUCCESS from here makes the code organization readable/easier from 1507 * the generic code. 1508 */ 1509 /*ARGSUSED*/ 1510 int 1511 px_lib_map_vconfig(dev_info_t *dip, 1512 ddi_map_req_t *mp, pci_config_offset_t off, 1513 pci_regspec_t *rp, caddr_t *addrp) 1514 { 1515 int fmcap; 1516 ndi_err_t *errp; 1517 on_trap_data_t *otp; 1518 ddi_acc_hdl_t *hp; 1519 ddi_acc_impl_t *ap; 1520 uchar_t busnum; /* bus number */ 1521 uchar_t devnum; /* device number */ 1522 uchar_t funcnum; /* function number */ 1523 px_config_acc_pvt_t *px_pvt; 1524 1525 hp = (ddi_acc_hdl_t *)mp->map_handlep; 1526 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1527 1528 /* Check for mapping teardown operation */ 1529 if ((mp->map_op == DDI_MO_UNMAP) || 1530 (mp->map_op == DDI_MO_UNLOCK)) { 1531 /* free up memory allocated for the private access handle. */ 1532 px_pvt = (px_config_acc_pvt_t *)hp->ah_bus_private; 1533 kmem_free((void *)px_pvt, sizeof (px_config_acc_pvt_t)); 1534 1535 /* unmap operation of PCI IO/config space. */ 1536 return (DDI_SUCCESS); 1537 } 1538 1539 fmcap = ddi_fm_capable(dip); 1540 if (DDI_FM_ACC_ERR_CAP(fmcap)) { 1541 errp = ((ddi_acc_impl_t *)hp)->ahi_err; 1542 otp = (on_trap_data_t *)errp->err_ontrap; 1543 otp->ot_handle = (void *)(hp); 1544 otp->ot_prot = OT_DATA_ACCESS; 1545 errp->err_status = DDI_FM_OK; 1546 errp->err_expected = DDI_FM_ERR_UNEXPECTED; 1547 errp->err_cf = px_err_cfg_hdl_check; 1548 } 1549 1550 ap->ahi_get8 = px_pci_config_get8; 1551 ap->ahi_get16 = px_pci_config_get16; 1552 ap->ahi_get32 = px_pci_config_get32; 1553 ap->ahi_get64 = px_pci_config_get64; 1554 ap->ahi_put8 = px_pci_config_put8; 1555 ap->ahi_put16 = px_pci_config_put16; 1556 ap->ahi_put32 = px_pci_config_put32; 1557 ap->ahi_put64 = px_pci_config_put64; 1558 ap->ahi_rep_get8 = px_pci_config_rep_get8; 1559 ap->ahi_rep_get16 = px_pci_config_rep_get16; 1560 ap->ahi_rep_get32 = px_pci_config_rep_get32; 1561 ap->ahi_rep_get64 = px_pci_config_rep_get64; 1562 ap->ahi_rep_put8 = px_pci_config_rep_put8; 1563 ap->ahi_rep_put16 = px_pci_config_rep_put16; 1564 ap->ahi_rep_put32 = px_pci_config_rep_put32; 1565 ap->ahi_rep_put64 = px_pci_config_rep_put64; 1566 1567 /* Initialize to default check/notify functions */ 1568 ap->ahi_fault = 0; 1569 ap->ahi_fault_check = i_ddi_acc_fault_check; 1570 ap->ahi_fault_notify = i_ddi_acc_fault_notify; 1571 1572 /* allocate memory for our private handle */ 1573 px_pvt = (px_config_acc_pvt_t *) 1574 kmem_zalloc(sizeof (px_config_acc_pvt_t), KM_SLEEP); 1575 hp->ah_bus_private = (void *)px_pvt; 1576 1577 busnum = PCI_REG_BUS_G(rp->pci_phys_hi); 1578 devnum = PCI_REG_DEV_G(rp->pci_phys_hi); 1579 funcnum = PCI_REG_FUNC_G(rp->pci_phys_hi); 1580 1581 /* set up private data for use during IO routines */ 1582 1583 /* addr needed by the HV APIs */ 1584 px_pvt->raddr = busnum << 16 | devnum << 11 | funcnum << 8; 1585 /* 1586 * Address that specifies the actual offset into the 256MB 1587 * memory mapped configuration space, 4K per device. 1588 * First 12bits form the offset into 4K config space. 1589 * This address is only used during the IO routines to calculate 1590 * the offset at which the transaction must be performed. 1591 * Drivers bypassing DDI functions to access PCI config space will 1592 * panic the system since the following is a bogus virtual address. 1593 */ 1594 px_pvt->vaddr = busnum << 20 | devnum << 15 | funcnum << 12 | off; 1595 px_pvt->dip = dip; 1596 1597 DBG(DBG_LIB_CFG, dip, "px_config_setup: raddr 0x%x, vaddr 0x%x\n", 1598 px_pvt->raddr, px_pvt->vaddr); 1599 *addrp = (caddr_t)(uintptr_t)px_pvt->vaddr; 1600 return (DDI_SUCCESS); 1601 } 1602 1603 /*ARGSUSED*/ 1604 void 1605 px_lib_map_attr_check(ddi_map_req_t *mp) 1606 { 1607 } 1608 1609 /* 1610 * px_lib_log_safeacc_err: 1611 * Imitate a cpu/mem trap call when a peek/poke fails. 1612 * This will initiate something similar to px_fm_callback. 1613 */ 1614 static void 1615 px_lib_log_safeacc_err(px_t *px_p, ddi_acc_handle_t handle, int fme_flag, 1616 r_addr_t addr) 1617 { 1618 uint32_t addr_high, addr_low; 1619 pcie_req_id_t bdf = PCIE_INVALID_BDF; 1620 pci_ranges_t *ranges_p; 1621 int range_len, i; 1622 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle; 1623 ddi_fm_error_t derr; 1624 1625 if (px_fm_enter(px_p) != DDI_SUCCESS) 1626 return; 1627 1628 derr.fme_status = DDI_FM_NONFATAL; 1629 derr.fme_version = DDI_FME_VERSION; 1630 derr.fme_flag = fme_flag; 1631 derr.fme_ena = fm_ena_generate(0, FM_ENA_FMT1); 1632 derr.fme_acc_handle = handle; 1633 if (hp) 1634 hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED; 1635 1636 addr_high = (uint32_t)(addr >> 32); 1637 addr_low = (uint32_t)addr; 1638 1639 /* 1640 * Make sure this failed load came from this PCIe port. Check by 1641 * matching the upper 32 bits of the address with the ranges property. 1642 */ 1643 range_len = px_p->px_ranges_length / sizeof (pci_ranges_t); 1644 i = 0; 1645 for (ranges_p = px_p->px_ranges_p; i < range_len; i++, ranges_p++) { 1646 if (ranges_p->parent_high == addr_high) { 1647 switch (ranges_p->child_high & PCI_ADDR_MASK) { 1648 case PCI_ADDR_CONFIG: 1649 bdf = (pcie_req_id_t)(addr_low >> 12); 1650 break; 1651 default: 1652 bdf = PCIE_INVALID_BDF; 1653 break; 1654 } 1655 break; 1656 } 1657 } 1658 1659 (void) px_rp_en_q(px_p, bdf, addr, NULL); 1660 (void) px_scan_fabric(px_p, px_p->px_dip, &derr); 1661 px_fm_exit(px_p); 1662 } 1663 1664 1665 #ifdef DEBUG 1666 int px_peekfault_cnt = 0; 1667 int px_pokefault_cnt = 0; 1668 #endif /* DEBUG */ 1669 1670 /* 1671 * Do a safe write to a device. 1672 * 1673 * When this function is given a handle (cautious access), all errors are 1674 * suppressed. 1675 * 1676 * When this function is not given a handle (poke), only Unsupported Request 1677 * and Completer Abort errors are suppressed. 1678 * 1679 * In all cases, all errors are returned in the function return status. 1680 */ 1681 1682 int 1683 px_lib_ctlops_poke(dev_info_t *dip, dev_info_t *rdip, 1684 peekpoke_ctlops_t *in_args) 1685 { 1686 px_t *px_p = DIP_TO_STATE(dip); 1687 px_pec_t *pec_p = px_p->px_pec_p; 1688 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle; 1689 1690 size_t repcount = in_args->repcount; 1691 size_t size = in_args->size; 1692 uintptr_t dev_addr = in_args->dev_addr; 1693 uintptr_t host_addr = in_args->host_addr; 1694 1695 int err = DDI_SUCCESS; 1696 uint64_t hvio_poke_status; 1697 uint32_t wrt_stat; 1698 1699 r_addr_t ra; 1700 uint64_t pokeval; 1701 pcie_req_id_t bdf; 1702 1703 ra = (r_addr_t)va_to_pa((void *)dev_addr); 1704 for (; repcount; repcount--) { 1705 1706 switch (size) { 1707 case sizeof (uint8_t): 1708 pokeval = *(uint8_t *)host_addr; 1709 break; 1710 case sizeof (uint16_t): 1711 pokeval = *(uint16_t *)host_addr; 1712 break; 1713 case sizeof (uint32_t): 1714 pokeval = *(uint32_t *)host_addr; 1715 break; 1716 case sizeof (uint64_t): 1717 pokeval = *(uint64_t *)host_addr; 1718 break; 1719 default: 1720 DBG(DBG_MAP, px_p->px_dip, 1721 "poke: invalid size %d passed\n", size); 1722 err = DDI_FAILURE; 1723 goto done; 1724 } 1725 1726 /* 1727 * Grab pokefault mutex since hypervisor does not guarantee 1728 * poke serialization. 1729 */ 1730 if (hp) { 1731 i_ndi_busop_access_enter(hp->ahi_common.ah_dip, 1732 (ddi_acc_handle_t)hp); 1733 pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED; 1734 } else { 1735 mutex_enter(&pec_p->pec_pokefault_mutex); 1736 pec_p->pec_safeacc_type = DDI_FM_ERR_POKE; 1737 } 1738 1739 if (pcie_get_bdf_from_dip(rdip, &bdf) != DDI_SUCCESS) { 1740 err = DDI_FAILURE; 1741 goto done; 1742 } 1743 1744 hvio_poke_status = hvio_poke(px_p->px_dev_hdl, ra, size, 1745 pokeval, bdf << 8, &wrt_stat); 1746 1747 if ((hvio_poke_status != H_EOK) || (wrt_stat != H_EOK)) { 1748 err = DDI_FAILURE; 1749 #ifdef DEBUG 1750 px_pokefault_cnt++; 1751 #endif 1752 /* 1753 * For CAUTIOUS and POKE access, notify FMA to 1754 * cleanup. Imitate a cpu/mem trap call like in sun4u. 1755 */ 1756 px_lib_log_safeacc_err(px_p, (ddi_acc_handle_t)hp, 1757 (hp ? DDI_FM_ERR_EXPECTED : 1758 DDI_FM_ERR_POKE), ra); 1759 1760 pec_p->pec_ontrap_data = NULL; 1761 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1762 if (hp) { 1763 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, 1764 (ddi_acc_handle_t)hp); 1765 } else { 1766 mutex_exit(&pec_p->pec_pokefault_mutex); 1767 } 1768 goto done; 1769 } 1770 1771 pec_p->pec_ontrap_data = NULL; 1772 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1773 if (hp) { 1774 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, 1775 (ddi_acc_handle_t)hp); 1776 } else { 1777 mutex_exit(&pec_p->pec_pokefault_mutex); 1778 } 1779 1780 host_addr += size; 1781 1782 if (in_args->flags == DDI_DEV_AUTOINCR) { 1783 dev_addr += size; 1784 ra = (r_addr_t)va_to_pa((void *)dev_addr); 1785 } 1786 } 1787 1788 done: 1789 return (err); 1790 } 1791 1792 1793 /*ARGSUSED*/ 1794 int 1795 px_lib_ctlops_peek(dev_info_t *dip, dev_info_t *rdip, 1796 peekpoke_ctlops_t *in_args, void *result) 1797 { 1798 px_t *px_p = DIP_TO_STATE(dip); 1799 px_pec_t *pec_p = px_p->px_pec_p; 1800 ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle; 1801 1802 size_t repcount = in_args->repcount; 1803 uintptr_t dev_addr = in_args->dev_addr; 1804 uintptr_t host_addr = in_args->host_addr; 1805 1806 r_addr_t ra; 1807 uint32_t read_status; 1808 uint64_t hvio_peek_status; 1809 uint64_t peekval; 1810 int err = DDI_SUCCESS; 1811 1812 result = (void *)in_args->host_addr; 1813 1814 ra = (r_addr_t)va_to_pa((void *)dev_addr); 1815 for (; repcount; repcount--) { 1816 1817 /* Lock pokefault mutex so read doesn't mask a poke fault. */ 1818 if (hp) { 1819 i_ndi_busop_access_enter(hp->ahi_common.ah_dip, 1820 (ddi_acc_handle_t)hp); 1821 pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED; 1822 } else { 1823 mutex_enter(&pec_p->pec_pokefault_mutex); 1824 pec_p->pec_safeacc_type = DDI_FM_ERR_PEEK; 1825 } 1826 1827 hvio_peek_status = hvio_peek(px_p->px_dev_hdl, ra, 1828 in_args->size, &read_status, &peekval); 1829 1830 if ((hvio_peek_status != H_EOK) || (read_status != H_EOK)) { 1831 err = DDI_FAILURE; 1832 1833 /* 1834 * For CAUTIOUS and PEEK access, notify FMA to 1835 * cleanup. Imitate a cpu/mem trap call like in sun4u. 1836 */ 1837 px_lib_log_safeacc_err(px_p, (ddi_acc_handle_t)hp, 1838 (hp ? DDI_FM_ERR_EXPECTED : 1839 DDI_FM_ERR_PEEK), ra); 1840 1841 /* Stuff FFs in host addr if peek. */ 1842 if (hp == NULL) { 1843 int i; 1844 uint8_t *ff_addr = (uint8_t *)host_addr; 1845 for (i = 0; i < in_args->size; i++) 1846 *ff_addr++ = 0xff; 1847 } 1848 #ifdef DEBUG 1849 px_peekfault_cnt++; 1850 #endif 1851 pec_p->pec_ontrap_data = NULL; 1852 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1853 if (hp) { 1854 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, 1855 (ddi_acc_handle_t)hp); 1856 } else { 1857 mutex_exit(&pec_p->pec_pokefault_mutex); 1858 } 1859 goto done; 1860 1861 } 1862 pec_p->pec_ontrap_data = NULL; 1863 pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED; 1864 if (hp) { 1865 i_ndi_busop_access_exit(hp->ahi_common.ah_dip, 1866 (ddi_acc_handle_t)hp); 1867 } else { 1868 mutex_exit(&pec_p->pec_pokefault_mutex); 1869 } 1870 1871 switch (in_args->size) { 1872 case sizeof (uint8_t): 1873 *(uint8_t *)host_addr = (uint8_t)peekval; 1874 break; 1875 case sizeof (uint16_t): 1876 *(uint16_t *)host_addr = (uint16_t)peekval; 1877 break; 1878 case sizeof (uint32_t): 1879 *(uint32_t *)host_addr = (uint32_t)peekval; 1880 break; 1881 case sizeof (uint64_t): 1882 *(uint64_t *)host_addr = (uint64_t)peekval; 1883 break; 1884 default: 1885 DBG(DBG_MAP, px_p->px_dip, 1886 "peek: invalid size %d passed\n", 1887 in_args->size); 1888 err = DDI_FAILURE; 1889 goto done; 1890 } 1891 1892 host_addr += in_args->size; 1893 1894 if (in_args->flags == DDI_DEV_AUTOINCR) { 1895 dev_addr += in_args->size; 1896 ra = (r_addr_t)va_to_pa((void *)dev_addr); 1897 } 1898 } 1899 done: 1900 return (err); 1901 } 1902 1903 1904 /* add interrupt vector */ 1905 int 1906 px_err_add_intr(px_fault_t *px_fault_p) 1907 { 1908 px_t *px_p = DIP_TO_STATE(px_fault_p->px_fh_dip); 1909 1910 DBG(DBG_LIB_INT, px_p->px_dip, 1911 "px_err_add_intr: calling add_ivintr"); 1912 1913 VERIFY(add_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL, 1914 (intrfunc)px_fault_p->px_err_func, (caddr_t)px_fault_p, NULL, 1915 (caddr_t)&px_fault_p->px_intr_payload[0]) == 0); 1916 1917 DBG(DBG_LIB_INT, px_p->px_dip, 1918 "px_err_add_intr: ib_intr_enable "); 1919 1920 px_ib_intr_enable(px_p, intr_dist_cpuid(), px_fault_p->px_intr_ino); 1921 1922 return (DDI_SUCCESS); 1923 } 1924 1925 /* remove interrupt vector */ 1926 void 1927 px_err_rem_intr(px_fault_t *px_fault_p) 1928 { 1929 px_t *px_p = DIP_TO_STATE(px_fault_p->px_fh_dip); 1930 1931 px_ib_intr_disable(px_p->px_ib_p, px_fault_p->px_intr_ino, 1932 IB_INTR_WAIT); 1933 1934 VERIFY(rem_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL) == 0); 1935 } 1936 1937 void 1938 px_cb_intr_redist(void *arg) 1939 { 1940 px_t *px_p = (px_t *)arg; 1941 px_ib_intr_dist_en(px_p->px_dip, intr_dist_cpuid(), 1942 px_p->px_inos[PX_INTR_XBC], B_FALSE); 1943 } 1944 1945 int 1946 px_cb_add_intr(px_fault_t *f_p) 1947 { 1948 px_t *px_p = DIP_TO_STATE(f_p->px_fh_dip); 1949 1950 DBG(DBG_LIB_INT, px_p->px_dip, 1951 "px_err_add_intr: calling add_ivintr"); 1952 1953 VERIFY(add_ivintr(f_p->px_fh_sysino, PX_ERR_PIL, 1954 (intrfunc)f_p->px_err_func, (caddr_t)f_p, NULL, 1955 (caddr_t)&f_p->px_intr_payload[0]) == 0); 1956 1957 intr_dist_add(px_cb_intr_redist, px_p); 1958 1959 DBG(DBG_LIB_INT, px_p->px_dip, 1960 "px_err_add_intr: ib_intr_enable "); 1961 1962 px_ib_intr_enable(px_p, intr_dist_cpuid(), f_p->px_intr_ino); 1963 1964 return (DDI_SUCCESS); 1965 } 1966 1967 void 1968 px_cb_rem_intr(px_fault_t *f_p) 1969 { 1970 intr_dist_rem(px_cb_intr_redist, DIP_TO_STATE(f_p->px_fh_dip)); 1971 px_err_rem_intr(f_p); 1972 } 1973 1974 #ifdef FMA 1975 void 1976 px_fill_rc_status(px_fault_t *px_fault_p, pciex_rc_error_regs_t *rc_status) 1977 { 1978 px_pec_err_t *err_pkt; 1979 1980 err_pkt = (px_pec_err_t *)px_fault_p->px_intr_payload; 1981 1982 /* initialise all the structure members */ 1983 rc_status->status_valid = 0; 1984 1985 if (err_pkt->pec_descr.P) { 1986 /* PCI Status Register */ 1987 rc_status->pci_err_status = err_pkt->pci_err_status; 1988 rc_status->status_valid |= PCI_ERR_STATUS_VALID; 1989 } 1990 1991 if (err_pkt->pec_descr.E) { 1992 /* PCIe Status Register */ 1993 rc_status->pcie_err_status = err_pkt->pcie_err_status; 1994 rc_status->status_valid |= PCIE_ERR_STATUS_VALID; 1995 } 1996 1997 if (err_pkt->pec_descr.U) { 1998 rc_status->ue_status = err_pkt->ue_reg_status; 1999 rc_status->status_valid |= UE_STATUS_VALID; 2000 } 2001 2002 if (err_pkt->pec_descr.H) { 2003 rc_status->ue_hdr1 = err_pkt->hdr[0]; 2004 rc_status->status_valid |= UE_HDR1_VALID; 2005 } 2006 2007 if (err_pkt->pec_descr.I) { 2008 rc_status->ue_hdr2 = err_pkt->hdr[1]; 2009 rc_status->status_valid |= UE_HDR2_VALID; 2010 } 2011 2012 /* ue_fst_err_ptr - not available for sun4v?? */ 2013 2014 2015 if (err_pkt->pec_descr.S) { 2016 rc_status->source_id = err_pkt->err_src_reg; 2017 rc_status->status_valid |= SOURCE_ID_VALID; 2018 } 2019 2020 if (err_pkt->pec_descr.R) { 2021 rc_status->root_err_status = err_pkt->root_err_status; 2022 rc_status->status_valid |= CE_STATUS_VALID; 2023 } 2024 } 2025 #endif 2026 2027 /*ARGSUSED*/ 2028 int 2029 px_lib_pmctl(int cmd, px_t *px_p) 2030 { 2031 return (DDI_FAILURE); 2032 } 2033 2034 /*ARGSUSED*/ 2035 uint_t 2036 px_pmeq_intr(caddr_t arg) 2037 { 2038 return (DDI_INTR_CLAIMED); 2039 } 2040 2041 /* 2042 * fetch the config space base addr of the root complex 2043 * note this depends on px structure being initialized 2044 */ 2045 uint64_t 2046 px_lib_get_cfgacc_base(dev_info_t *dip) 2047 { 2048 int instance = DIP_TO_INST(dip); 2049 px_t *px_p = INST_TO_STATE(instance); 2050 2051 return (px_p->px_dev_hdl); 2052 } 2053 2054 void 2055 px_panic_domain(px_t *px_p, pcie_req_id_t bdf) 2056 { 2057 uint64_t ret; 2058 dev_info_t *dip = px_p->px_dip; 2059 2060 DBG(DBG_ERR_INTR, dip, "px_panic_domain: handle 0x%lx, ino %d, " 2061 "bdf<<8 0x%lx\n", 2062 (uint64_t)DIP_TO_HANDLE(dip), px_p->px_cb_fault.px_intr_ino, 2063 (pci_device_t)bdf << 8); 2064 if ((ret = pci_error_send(DIP_TO_HANDLE(dip), 2065 px_p->px_cb_fault.px_intr_ino, (pci_device_t)bdf << 8)) != H_EOK) { 2066 DBG(DBG_ERR_INTR, dip, "pci_error_send failed, ret 0x%lx\n", 2067 ret); 2068 } else 2069 DBG(DBG_ERR_INTR, dip, "pci_error_send worked\n"); 2070 } 2071 2072 /*ARGSUSED*/ 2073 int 2074 px_lib_hotplug_init(dev_info_t *dip, void *arg) 2075 { 2076 return (DDI_ENOTSUP); 2077 } 2078 2079 /*ARGSUSED*/ 2080 void 2081 px_lib_hotplug_uninit(dev_info_t *dip) 2082 { 2083 } 2084 2085 /*ARGSUSED*/ 2086 void 2087 px_hp_intr_redist(px_t *px_p) 2088 { 2089 } 2090 2091 /* Dummy cpr add callback */ 2092 /*ARGSUSED*/ 2093 void 2094 px_cpr_add_callb(px_t *px_p) 2095 { 2096 } 2097 2098 /* Dummy cpr rem callback */ 2099 /*ARGSUSED*/ 2100 void 2101 px_cpr_rem_callb(px_t *px_p) 2102 { 2103 } 2104 2105 /*ARGSUSED*/ 2106 boolean_t 2107 px_lib_is_in_drain_state(px_t *px_p) 2108 { 2109 return (B_FALSE); 2110 } 2111 2112 /* 2113 * There is no IOAPI to get the BDF of the pcie root port nexus at this moment. 2114 * Assume it is 0x0000, until otherwise noted. For now, all sun4v platforms 2115 * have programmed the BDF to be 0x0000. 2116 */ 2117 /*ARGSUSED*/ 2118 pcie_req_id_t 2119 px_lib_get_bdf(px_t *px_p) 2120 { 2121 return (0x0000); 2122 } 2123 2124 int 2125 px_lib_get_root_complex_mps(px_t *px_p, dev_info_t *dip, int *mps) 2126 { 2127 pci_device_t bdf = px_lib_get_bdf(px_p); 2128 2129 if (hvio_get_rp_mps_cap(DIP_TO_HANDLE(dip), bdf, mps) == H_EOK) 2130 return (DDI_SUCCESS); 2131 else 2132 return (DDI_FAILURE); 2133 } 2134 2135 int 2136 px_lib_set_root_complex_mps(px_t *px_p, dev_info_t *dip, int mps) 2137 { 2138 pci_device_t bdf = px_lib_get_bdf(px_p); 2139 2140 if (hvio_set_rp_mps(DIP_TO_HANDLE(dip), bdf, mps) == H_EOK) 2141 return (DDI_SUCCESS); 2142 else 2143 return (DDI_FAILURE); 2144 } 2145 2146 static int 2147 px_lib_do_count_waiting_dev(dev_info_t *dip, void *arg) 2148 { 2149 int *count = (int *)arg; 2150 dev_info_t *cdip = ddi_get_child(dip); 2151 2152 while (cdip != NULL) { 2153 /* check if this is an assigned device */ 2154 if (ddi_prop_exists(DDI_DEV_T_NONE, cdip, DDI_PROP_DONTPASS, 2155 "ddi-assigned")) { 2156 DBG(DBG_ATTACH, dip, "px_lib_do_count_waiting_dev: " 2157 "Found an assigned dev %p, under bridge %p", 2158 cdip, dip); 2159 2160 /* 2161 * Mark this bridge as needing waiting for 2162 * CHILD_LOANED will be removed after bridge reports 2163 * its readyness back to px driver 2164 */ 2165 if (ddi_prop_update_int(DDI_DEV_T_NONE, dip, 2166 CHILD_LOANED, 1) == DDI_PROP_SUCCESS) 2167 (*count)++; 2168 break; 2169 } 2170 cdip = ddi_get_next_sibling(cdip); 2171 } 2172 2173 return (DDI_WALK_CONTINUE); 2174 } 2175 2176 static int 2177 px_lib_count_waiting_dev(dev_info_t *dip) 2178 { 2179 int circular_count; 2180 int count = 0; 2181 2182 /* No need to continue if this system is not SDIO capable */ 2183 if (px_sdio_users == 0) 2184 return (0); 2185 2186 /* see if px iteslf has assigned children */ 2187 (void) px_lib_do_count_waiting_dev(dip, &count); 2188 2189 /* scan dev under this px */ 2190 ndi_devi_enter(dip, &circular_count); 2191 ddi_walk_devs(ddi_get_child(dip), px_lib_do_count_waiting_dev, &count); 2192 ndi_devi_exit(dip, circular_count); 2193 return (count); 2194 } 2195 2196 /* Called from px/bridge driver directly to report its readyness */ 2197 int 2198 px_lib_fabric_sync(dev_info_t *dip) 2199 { 2200 px_t *px; 2201 dev_info_t *rcdip; 2202 int waitdev; 2203 2204 /* No need to continue if this system is not SDIO capable */ 2205 if (px_sdio_users == 0) 2206 return (DDI_SUCCESS); 2207 2208 /* a valid bridge w/ assigned dev under it? */ 2209 if (ddi_prop_remove(DDI_DEV_T_NONE, dip, CHILD_LOANED) != 2210 DDI_PROP_SUCCESS) 2211 return (DDI_FAILURE); 2212 2213 /* find out RC dip */ 2214 for (rcdip = dip; rcdip != NULL; rcdip = ddi_get_parent(rcdip)) { 2215 if (PCIE_DIP2BUS(rcdip) && PCIE_IS_RC(PCIE_DIP2BUS(rcdip))) 2216 break; 2217 } 2218 if ((rcdip == NULL) || ((px = (px_t *)DIP_TO_STATE(rcdip)) == NULL)) 2219 return (DDI_FAILURE); 2220 2221 /* are we ready? */ 2222 waitdev = (int)(uintptr_t)px->px_plat_p; 2223 ASSERT(waitdev); 2224 DBG(DBG_CTLOPS, rcdip, "px_lib_fabric_sync: " 2225 "Px/bridge %p is ready, %d left", rcdip, waitdev - 1); 2226 --waitdev; 2227 px->px_plat_p = (void *)(uintptr_t)waitdev; 2228 if (waitdev != 0) 2229 return (DDI_SUCCESS); 2230 2231 /* notify hpyervisor */ 2232 DBG(DBG_CTLOPS, rcdip, "px_lib_fabric_sync: " 2233 "Notifying HV that RC %p is ready users=%d", rcdip, px_sdio_users); 2234 2235 if (pci_iov_root_configured(px->px_dev_hdl) != H_EOK) 2236 return (DDI_FAILURE); 2237 2238 return (DDI_SUCCESS); 2239 } 2240