1 /*- 2 * Copyright (c) 2015, 2020 Ruslan Bukin <br@bsdpad.com> 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * This software was developed by Semihalf under 7 * the sponsorship of the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /* Generic ECAM PCIe driver */ 32 33 #include <sys/cdefs.h> 34 #include "opt_platform.h" 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/malloc.h> 39 #include <sys/kernel.h> 40 #include <sys/rman.h> 41 #include <sys/module.h> 42 #include <sys/bus.h> 43 #include <sys/endian.h> 44 45 #include <dev/pci/pcivar.h> 46 #include <dev/pci/pcireg.h> 47 #include <dev/pci/pcib_private.h> 48 #include <dev/pci/pci_host_generic.h> 49 50 #include <machine/bus.h> 51 #include <machine/intr.h> 52 53 #include "pcib_if.h" 54 55 #if defined(VM_MEMATTR_DEVICE_NP) 56 #define PCI_UNMAPPED 57 #define PCI_RF_FLAGS RF_UNMAPPED 58 #else 59 #define PCI_RF_FLAGS 0 60 #endif 61 62 63 /* Forward prototypes */ 64 65 static uint32_t generic_pcie_read_config(device_t dev, u_int bus, u_int slot, 66 u_int func, u_int reg, int bytes); 67 static void generic_pcie_write_config(device_t dev, u_int bus, u_int slot, 68 u_int func, u_int reg, uint32_t val, int bytes); 69 static int generic_pcie_maxslots(device_t dev); 70 static int generic_pcie_read_ivar(device_t dev, device_t child, int index, 71 uintptr_t *result); 72 static int generic_pcie_write_ivar(device_t dev, device_t child, int index, 73 uintptr_t value); 74 75 int 76 pci_host_generic_core_attach(device_t dev) 77 { 78 #ifdef PCI_UNMAPPED 79 struct resource_map_request req; 80 struct resource_map map; 81 #endif 82 struct generic_pcie_core_softc *sc; 83 uint64_t phys_base; 84 uint64_t pci_base; 85 uint64_t size; 86 int domain, error; 87 int rid, tuple; 88 89 sc = device_get_softc(dev); 90 sc->dev = dev; 91 92 /* Create the parent DMA tag to pass down the coherent flag */ 93 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 94 1, 0, /* alignment, bounds */ 95 BUS_SPACE_MAXADDR, /* lowaddr */ 96 BUS_SPACE_MAXADDR, /* highaddr */ 97 NULL, NULL, /* filter, filterarg */ 98 BUS_SPACE_MAXSIZE, /* maxsize */ 99 BUS_SPACE_UNRESTRICTED, /* nsegments */ 100 BUS_SPACE_MAXSIZE, /* maxsegsize */ 101 sc->coherent ? BUS_DMA_COHERENT : 0, /* flags */ 102 NULL, NULL, /* lockfunc, lockarg */ 103 &sc->dmat); 104 if (error != 0) 105 return (error); 106 107 /* 108 * Attempt to set the domain. If it's missing, or we are unable to 109 * set it then memory allocations may be placed in the wrong domain. 110 */ 111 if (bus_get_domain(dev, &domain) == 0) 112 (void)bus_dma_tag_set_domain(sc->dmat, domain); 113 114 if ((sc->quirks & PCIE_CUSTOM_CONFIG_SPACE_QUIRK) == 0) { 115 rid = 0; 116 sc->res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 117 PCI_RF_FLAGS | RF_ACTIVE); 118 if (sc->res == NULL) { 119 device_printf(dev, "could not allocate memory.\n"); 120 error = ENXIO; 121 goto err_resource; 122 } 123 #ifdef PCI_UNMAPPED 124 resource_init_map_request(&req); 125 req.memattr = VM_MEMATTR_DEVICE_NP; 126 error = bus_map_resource(dev, SYS_RES_MEMORY, sc->res, &req, 127 &map); 128 if (error != 0) { 129 device_printf(dev, "could not map memory.\n"); 130 return (error); 131 } 132 rman_set_mapping(sc->res, &map); 133 #endif 134 } 135 136 sc->has_pmem = false; 137 sc->pmem_rman.rm_type = RMAN_ARRAY; 138 sc->pmem_rman.rm_descr = "PCIe Prefetch Memory"; 139 140 sc->mem_rman.rm_type = RMAN_ARRAY; 141 sc->mem_rman.rm_descr = "PCIe Memory"; 142 143 sc->io_rman.rm_type = RMAN_ARRAY; 144 sc->io_rman.rm_descr = "PCIe IO window"; 145 146 /* Initialize rman and allocate memory regions */ 147 error = rman_init(&sc->pmem_rman); 148 if (error) { 149 device_printf(dev, "rman_init() failed. error = %d\n", error); 150 goto err_pmem_rman; 151 } 152 153 error = rman_init(&sc->mem_rman); 154 if (error) { 155 device_printf(dev, "rman_init() failed. error = %d\n", error); 156 goto err_mem_rman; 157 } 158 159 error = rman_init(&sc->io_rman); 160 if (error) { 161 device_printf(dev, "rman_init() failed. error = %d\n", error); 162 goto err_io_rman; 163 } 164 165 for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { 166 phys_base = sc->ranges[tuple].phys_base; 167 pci_base = sc->ranges[tuple].pci_base; 168 size = sc->ranges[tuple].size; 169 if (phys_base == 0 || size == 0) 170 continue; /* empty range element */ 171 switch (FLAG_TYPE(sc->ranges[tuple].flags)) { 172 case FLAG_TYPE_PMEM: 173 sc->has_pmem = true; 174 error = rman_manage_region(&sc->pmem_rman, 175 pci_base, pci_base + size - 1); 176 break; 177 case FLAG_TYPE_MEM: 178 error = rman_manage_region(&sc->mem_rman, 179 pci_base, pci_base + size - 1); 180 break; 181 case FLAG_TYPE_IO: 182 error = rman_manage_region(&sc->io_rman, 183 pci_base, pci_base + size - 1); 184 break; 185 default: 186 continue; 187 } 188 if (error) { 189 device_printf(dev, "rman_manage_region() failed." 190 "error = %d\n", error); 191 goto err_rman_manage; 192 } 193 } 194 195 return (0); 196 197 err_rman_manage: 198 rman_fini(&sc->io_rman); 199 err_io_rman: 200 rman_fini(&sc->mem_rman); 201 err_mem_rman: 202 rman_fini(&sc->pmem_rman); 203 err_pmem_rman: 204 if (sc->res != NULL) 205 bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->res); 206 err_resource: 207 bus_dma_tag_destroy(sc->dmat); 208 return (error); 209 } 210 211 int 212 pci_host_generic_core_detach(device_t dev) 213 { 214 struct generic_pcie_core_softc *sc; 215 int error; 216 217 sc = device_get_softc(dev); 218 219 error = bus_generic_detach(dev); 220 if (error != 0) 221 return (error); 222 223 rman_fini(&sc->io_rman); 224 rman_fini(&sc->mem_rman); 225 rman_fini(&sc->pmem_rman); 226 if (sc->res != NULL) 227 bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->res); 228 bus_dma_tag_destroy(sc->dmat); 229 230 return (0); 231 } 232 233 static uint32_t 234 generic_pcie_read_config(device_t dev, u_int bus, u_int slot, 235 u_int func, u_int reg, int bytes) 236 { 237 struct generic_pcie_core_softc *sc; 238 uint64_t offset; 239 uint32_t data; 240 241 sc = device_get_softc(dev); 242 if ((bus < sc->bus_start) || (bus > sc->bus_end)) 243 return (~0U); 244 if ((slot > PCI_SLOTMAX) || (func > PCI_FUNCMAX) || 245 (reg > PCIE_REGMAX)) 246 return (~0U); 247 if ((sc->quirks & PCIE_ECAM_DESIGNWARE_QUIRK) && bus == 0 && slot > 0) 248 return (~0U); 249 250 offset = PCIE_ADDR_OFFSET(bus - sc->bus_start, slot, func, reg); 251 252 switch (bytes) { 253 case 1: 254 data = bus_read_1(sc->res, offset); 255 break; 256 case 2: 257 data = le16toh(bus_read_2(sc->res, offset)); 258 break; 259 case 4: 260 data = le32toh(bus_read_4(sc->res, offset)); 261 break; 262 default: 263 return (~0U); 264 } 265 266 return (data); 267 } 268 269 static void 270 generic_pcie_write_config(device_t dev, u_int bus, u_int slot, 271 u_int func, u_int reg, uint32_t val, int bytes) 272 { 273 struct generic_pcie_core_softc *sc; 274 uint64_t offset; 275 276 sc = device_get_softc(dev); 277 if ((bus < sc->bus_start) || (bus > sc->bus_end)) 278 return; 279 if ((slot > PCI_SLOTMAX) || (func > PCI_FUNCMAX) || 280 (reg > PCIE_REGMAX)) 281 return; 282 283 offset = PCIE_ADDR_OFFSET(bus - sc->bus_start, slot, func, reg); 284 285 switch (bytes) { 286 case 1: 287 bus_write_1(sc->res, offset, val); 288 break; 289 case 2: 290 bus_write_2(sc->res, offset, htole16(val)); 291 break; 292 case 4: 293 bus_write_4(sc->res, offset, htole32(val)); 294 break; 295 default: 296 return; 297 } 298 } 299 300 static int 301 generic_pcie_maxslots(device_t dev) 302 { 303 304 return (31); /* max slots per bus acc. to standard */ 305 } 306 307 static int 308 generic_pcie_read_ivar(device_t dev, device_t child, int index, 309 uintptr_t *result) 310 { 311 struct generic_pcie_core_softc *sc; 312 313 sc = device_get_softc(dev); 314 315 if (index == PCIB_IVAR_BUS) { 316 *result = sc->bus_start; 317 return (0); 318 } 319 320 if (index == PCIB_IVAR_DOMAIN) { 321 *result = sc->ecam; 322 return (0); 323 } 324 325 if (bootverbose) 326 device_printf(dev, "ERROR: Unknown index %d.\n", index); 327 return (ENOENT); 328 } 329 330 static int 331 generic_pcie_write_ivar(device_t dev, device_t child, int index, 332 uintptr_t value) 333 { 334 335 return (ENOENT); 336 } 337 338 static struct rman * 339 generic_pcie_rman(struct generic_pcie_core_softc *sc, int type, int flags) 340 { 341 342 switch (type) { 343 case SYS_RES_IOPORT: 344 return (&sc->io_rman); 345 case SYS_RES_MEMORY: 346 if (sc->has_pmem && (flags & RF_PREFETCHABLE) != 0) 347 return (&sc->pmem_rman); 348 return (&sc->mem_rman); 349 default: 350 break; 351 } 352 353 return (NULL); 354 } 355 356 int 357 pci_host_generic_core_release_resource(device_t dev, device_t child, int type, 358 int rid, struct resource *res) 359 { 360 struct generic_pcie_core_softc *sc; 361 struct rman *rm; 362 int error; 363 364 sc = device_get_softc(dev); 365 366 #if defined(NEW_PCIB) && defined(PCI_RES_BUS) 367 if (type == PCI_RES_BUS) { 368 return (pci_domain_release_bus(sc->ecam, child, rid, res)); 369 } 370 #endif 371 372 rm = generic_pcie_rman(sc, type, rman_get_flags(res)); 373 if (rm != NULL) { 374 KASSERT(rman_is_region_manager(res, rm), ("rman mismatch")); 375 if (rman_get_flags(res) & RF_ACTIVE) { 376 error = bus_deactivate_resource(child, type, rid, res); 377 if (error) 378 return (error); 379 } 380 return (rman_release_resource(res)); 381 } 382 383 return (bus_generic_release_resource(dev, child, type, rid, res)); 384 } 385 386 static int 387 generic_pcie_translate_resource_common(device_t dev, int type, rman_res_t start, 388 rman_res_t end, rman_res_t *new_start, rman_res_t *new_end) 389 { 390 struct generic_pcie_core_softc *sc; 391 uint64_t phys_base; 392 uint64_t pci_base; 393 uint64_t size; 394 int i, space; 395 bool found; 396 397 sc = device_get_softc(dev); 398 /* Translate the address from a PCI address to a physical address */ 399 switch (type) { 400 case SYS_RES_IOPORT: 401 case SYS_RES_MEMORY: 402 found = false; 403 for (i = 0; i < MAX_RANGES_TUPLES; i++) { 404 pci_base = sc->ranges[i].pci_base; 405 phys_base = sc->ranges[i].phys_base; 406 size = sc->ranges[i].size; 407 408 if (start < pci_base || start >= pci_base + size) 409 continue; 410 411 switch (FLAG_TYPE(sc->ranges[i].flags)) { 412 case FLAG_TYPE_MEM: 413 case FLAG_TYPE_PMEM: 414 space = SYS_RES_MEMORY; 415 break; 416 case FLAG_TYPE_IO: 417 space = SYS_RES_IOPORT; 418 break; 419 default: 420 space = -1; 421 continue; 422 } 423 424 if (type == space) { 425 *new_start = start - pci_base + phys_base; 426 *new_end = end - pci_base + phys_base; 427 found = true; 428 break; 429 } 430 } 431 break; 432 default: 433 /* No translation for non-memory types */ 434 *new_start = start; 435 *new_end = end; 436 found = true; 437 break; 438 } 439 440 return (found ? 0 : ENOENT); 441 } 442 443 static int 444 generic_pcie_translate_resource(device_t bus, int type, 445 rman_res_t start, rman_res_t *newstart) 446 { 447 rman_res_t newend; /* unused */ 448 449 return (generic_pcie_translate_resource_common( 450 bus, type, start, 0, newstart, &newend)); 451 } 452 453 struct resource * 454 pci_host_generic_core_alloc_resource(device_t dev, device_t child, int type, 455 int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) 456 { 457 struct generic_pcie_core_softc *sc; 458 struct resource *res; 459 struct rman *rm; 460 461 sc = device_get_softc(dev); 462 463 #if defined(NEW_PCIB) && defined(PCI_RES_BUS) 464 if (type == PCI_RES_BUS) { 465 return (pci_domain_alloc_bus(sc->ecam, child, rid, start, end, 466 count, flags)); 467 } 468 #endif 469 470 rm = generic_pcie_rman(sc, type, flags); 471 if (rm == NULL) 472 return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child, 473 type, rid, start, end, count, flags)); 474 475 if (bootverbose) { 476 device_printf(dev, 477 "rman_reserve_resource: start=%#jx, end=%#jx, count=%#jx\n", 478 start, end, count); 479 } 480 481 res = rman_reserve_resource(rm, start, end, count, flags, child); 482 if (res == NULL) 483 goto fail; 484 485 rman_set_rid(res, *rid); 486 487 if (flags & RF_ACTIVE) 488 if (bus_activate_resource(child, type, *rid, res)) { 489 rman_release_resource(res); 490 goto fail; 491 } 492 493 return (res); 494 495 fail: 496 device_printf(dev, "%s FAIL: type=%d, rid=%d, " 497 "start=%016jx, end=%016jx, count=%016jx, flags=%x\n", 498 __func__, type, *rid, start, end, count, flags); 499 500 return (NULL); 501 } 502 503 static int 504 generic_pcie_activate_resource(device_t dev, device_t child, int type, 505 int rid, struct resource *r) 506 { 507 rman_res_t start, end; 508 int res; 509 510 if ((res = rman_activate_resource(r)) != 0) 511 return (res); 512 513 start = rman_get_start(r); 514 end = rman_get_end(r); 515 res = generic_pcie_translate_resource_common(dev, type, start, end, 516 &start, &end); 517 if (res != 0) { 518 rman_deactivate_resource(r); 519 return (res); 520 } 521 rman_set_start(r, start); 522 rman_set_end(r, end); 523 524 return (BUS_ACTIVATE_RESOURCE(device_get_parent(dev), child, type, 525 rid, r)); 526 } 527 528 static int 529 generic_pcie_deactivate_resource(device_t dev, device_t child, int type, 530 int rid, struct resource *r) 531 { 532 int res; 533 534 if ((res = rman_deactivate_resource(r)) != 0) 535 return (res); 536 537 switch (type) { 538 case SYS_RES_IOPORT: 539 case SYS_RES_MEMORY: 540 case SYS_RES_IRQ: 541 res = BUS_DEACTIVATE_RESOURCE(device_get_parent(dev), child, 542 type, rid, r); 543 break; 544 default: 545 break; 546 } 547 548 return (res); 549 } 550 551 static int 552 generic_pcie_adjust_resource(device_t dev, device_t child, int type, 553 struct resource *res, rman_res_t start, rman_res_t end) 554 { 555 struct generic_pcie_core_softc *sc; 556 struct rman *rm; 557 558 sc = device_get_softc(dev); 559 #if defined(NEW_PCIB) && defined(PCI_RES_BUS) 560 if (type == PCI_RES_BUS) 561 return (pci_domain_adjust_bus(sc->ecam, child, res, start, 562 end)); 563 #endif 564 565 rm = generic_pcie_rman(sc, type, rman_get_flags(res)); 566 if (rm != NULL) 567 return (rman_adjust_resource(res, start, end)); 568 return (bus_generic_adjust_resource(dev, child, type, res, start, end)); 569 } 570 571 static bus_dma_tag_t 572 generic_pcie_get_dma_tag(device_t dev, device_t child) 573 { 574 struct generic_pcie_core_softc *sc; 575 576 sc = device_get_softc(dev); 577 return (sc->dmat); 578 } 579 580 static device_method_t generic_pcie_methods[] = { 581 DEVMETHOD(device_attach, pci_host_generic_core_attach), 582 DEVMETHOD(device_detach, pci_host_generic_core_detach), 583 584 DEVMETHOD(bus_read_ivar, generic_pcie_read_ivar), 585 DEVMETHOD(bus_write_ivar, generic_pcie_write_ivar), 586 DEVMETHOD(bus_alloc_resource, pci_host_generic_core_alloc_resource), 587 DEVMETHOD(bus_adjust_resource, generic_pcie_adjust_resource), 588 DEVMETHOD(bus_activate_resource, generic_pcie_activate_resource), 589 DEVMETHOD(bus_deactivate_resource, generic_pcie_deactivate_resource), 590 DEVMETHOD(bus_release_resource, pci_host_generic_core_release_resource), 591 DEVMETHOD(bus_translate_resource, generic_pcie_translate_resource), 592 DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), 593 DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), 594 595 DEVMETHOD(bus_get_dma_tag, generic_pcie_get_dma_tag), 596 597 /* pcib interface */ 598 DEVMETHOD(pcib_maxslots, generic_pcie_maxslots), 599 DEVMETHOD(pcib_read_config, generic_pcie_read_config), 600 DEVMETHOD(pcib_write_config, generic_pcie_write_config), 601 602 DEVMETHOD_END 603 }; 604 605 DEFINE_CLASS_0(pcib, generic_pcie_core_driver, 606 generic_pcie_methods, sizeof(struct generic_pcie_core_softc)); 607