1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org> 6 * Copyright (c) 2000, BSDi 7 * Copyright (c) 2004, Scott Long <scottl@freebsd.org> 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/bus.h> 38 #include <sys/lock.h> 39 #include <sys/kernel.h> 40 #include <sys/mutex.h> 41 #include <sys/malloc.h> 42 #include <sys/queue.h> 43 #include <sys/sysctl.h> 44 #include <dev/pci/pcivar.h> 45 #include <dev/pci/pcireg.h> 46 #include <machine/pci_cfgreg.h> 47 #include <machine/pc/bios.h> 48 49 #include <vm/vm.h> 50 #include <vm/vm_param.h> 51 #include <vm/vm_kern.h> 52 #include <vm/vm_extern.h> 53 #include <vm/pmap.h> 54 55 #define PRVERB(a) do { \ 56 if (bootverbose) \ 57 printf a ; \ 58 } while(0) 59 60 #define PCIE_CACHE 8 61 struct pcie_cfg_elem { 62 TAILQ_ENTRY(pcie_cfg_elem) elem; 63 vm_offset_t vapage; 64 vm_paddr_t papage; 65 }; 66 67 enum { 68 CFGMECH_NONE = 0, 69 CFGMECH_1, 70 CFGMECH_2, 71 CFGMECH_PCIE, 72 }; 73 74 SYSCTL_DECL(_hw_pci); 75 76 static TAILQ_HEAD(pcie_cfg_list, pcie_cfg_elem) pcie_list[MAXCPU]; 77 static uint64_t pcie_base; 78 static int pcie_minbus, pcie_maxbus; 79 static uint32_t pcie_badslots; 80 static int cfgmech; 81 static int devmax; 82 static struct mtx pcicfg_mtx; 83 static int mcfg_enable = 1; 84 SYSCTL_INT(_hw_pci, OID_AUTO, mcfg, CTLFLAG_RDTUN, &mcfg_enable, 0, 85 "Enable support for PCI-e memory mapped config access"); 86 87 static uint32_t pci_docfgregread(int bus, int slot, int func, int reg, 88 int bytes); 89 static int pcireg_cfgread(int bus, int slot, int func, int reg, int bytes); 90 static void pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes); 91 static int pcireg_cfgopen(void); 92 static int pciereg_cfgread(int bus, unsigned slot, unsigned func, 93 unsigned reg, unsigned bytes); 94 static void pciereg_cfgwrite(int bus, unsigned slot, unsigned func, 95 unsigned reg, int data, unsigned bytes); 96 97 /* 98 * Some BIOS writers seem to want to ignore the spec and put 99 * 0 in the intline rather than 255 to indicate none. Some use 100 * numbers in the range 128-254 to indicate something strange and 101 * apparently undocumented anywhere. Assume these are completely bogus 102 * and map them to 255, which means "none". 103 */ 104 static __inline int 105 pci_i386_map_intline(int line) 106 { 107 if (line == 0 || line >= 128) 108 return (PCI_INVALID_IRQ); 109 return (line); 110 } 111 112 static u_int16_t 113 pcibios_get_version(void) 114 { 115 struct bios_regs args; 116 117 if (PCIbios.ventry == 0) { 118 PRVERB(("pcibios: No call entry point\n")); 119 return (0); 120 } 121 args.eax = PCIBIOS_BIOS_PRESENT; 122 if (bios32(&args, PCIbios.ventry, GSEL(GCODE_SEL, SEL_KPL))) { 123 PRVERB(("pcibios: BIOS_PRESENT call failed\n")); 124 return (0); 125 } 126 if (args.edx != 0x20494350) { 127 PRVERB(("pcibios: BIOS_PRESENT didn't return 'PCI ' in edx\n")); 128 return (0); 129 } 130 return (args.ebx & 0xffff); 131 } 132 133 /* 134 * Initialise access to PCI configuration space 135 */ 136 int 137 pci_cfgregopen(void) 138 { 139 static int opened = 0; 140 uint64_t pciebar; 141 u_int16_t vid, did; 142 u_int16_t v; 143 144 if (opened) 145 return (1); 146 147 if (cfgmech == CFGMECH_NONE && pcireg_cfgopen() == 0) 148 return (0); 149 150 v = pcibios_get_version(); 151 if (v > 0) 152 PRVERB(("pcibios: BIOS version %x.%02x\n", (v & 0xff00) >> 8, 153 v & 0xff)); 154 mtx_init(&pcicfg_mtx, "pcicfg", NULL, MTX_SPIN); 155 opened = 1; 156 157 /* $PIR requires PCI BIOS 2.10 or greater. */ 158 if (v >= 0x0210) 159 pci_pir_open(); 160 161 if (cfgmech == CFGMECH_PCIE) 162 return (1); 163 164 /* 165 * Grope around in the PCI config space to see if this is a 166 * chipset that is capable of doing memory-mapped config cycles. 167 * This also implies that it can do PCIe extended config cycles. 168 */ 169 170 /* Check for supported chipsets */ 171 vid = pci_cfgregread(0, 0, 0, PCIR_VENDOR, 2); 172 did = pci_cfgregread(0, 0, 0, PCIR_DEVICE, 2); 173 switch (vid) { 174 case 0x8086: 175 switch (did) { 176 case 0x3590: 177 case 0x3592: 178 /* Intel 7520 or 7320 */ 179 pciebar = pci_cfgregread(0, 0, 0, 0xce, 2) << 16; 180 pcie_cfgregopen(pciebar, 0, 255); 181 break; 182 case 0x2580: 183 case 0x2584: 184 case 0x2590: 185 /* Intel 915, 925, or 915GM */ 186 pciebar = pci_cfgregread(0, 0, 0, 0x48, 4); 187 pcie_cfgregopen(pciebar, 0, 255); 188 break; 189 } 190 } 191 192 return(1); 193 } 194 195 static uint32_t 196 pci_docfgregread(int bus, int slot, int func, int reg, int bytes) 197 { 198 199 if (cfgmech == CFGMECH_PCIE && 200 (bus >= pcie_minbus && bus <= pcie_maxbus) && 201 (bus != 0 || !(1 << slot & pcie_badslots))) 202 return (pciereg_cfgread(bus, slot, func, reg, bytes)); 203 else 204 return (pcireg_cfgread(bus, slot, func, reg, bytes)); 205 } 206 207 /* 208 * Read configuration space register 209 */ 210 u_int32_t 211 pci_cfgregread(int bus, int slot, int func, int reg, int bytes) 212 { 213 uint32_t line; 214 215 /* 216 * Some BIOS writers seem to want to ignore the spec and put 217 * 0 in the intline rather than 255 to indicate none. The rest of 218 * the code uses 255 as an invalid IRQ. 219 */ 220 if (reg == PCIR_INTLINE && bytes == 1) { 221 line = pci_docfgregread(bus, slot, func, PCIR_INTLINE, 1); 222 return (pci_i386_map_intline(line)); 223 } 224 return (pci_docfgregread(bus, slot, func, reg, bytes)); 225 } 226 227 /* 228 * Write configuration space register 229 */ 230 void 231 pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes) 232 { 233 234 if (cfgmech == CFGMECH_PCIE && 235 (bus >= pcie_minbus && bus <= pcie_maxbus) && 236 (bus != 0 || !(1 << slot & pcie_badslots))) 237 pciereg_cfgwrite(bus, slot, func, reg, data, bytes); 238 else 239 pcireg_cfgwrite(bus, slot, func, reg, data, bytes); 240 } 241 242 /* 243 * Configuration space access using direct register operations 244 */ 245 246 /* enable configuration space accesses and return data port address */ 247 static int 248 pci_cfgenable(unsigned bus, unsigned slot, unsigned func, int reg, int bytes) 249 { 250 int dataport = 0; 251 252 if (bus <= PCI_BUSMAX 253 && slot < devmax 254 && func <= PCI_FUNCMAX 255 && (unsigned)reg <= PCI_REGMAX 256 && bytes != 3 257 && (unsigned)bytes <= 4 258 && (reg & (bytes - 1)) == 0) { 259 switch (cfgmech) { 260 case CFGMECH_PCIE: 261 case CFGMECH_1: 262 outl(CONF1_ADDR_PORT, (1U << 31) 263 | (bus << 16) | (slot << 11) 264 | (func << 8) | (reg & ~0x03)); 265 dataport = CONF1_DATA_PORT + (reg & 0x03); 266 break; 267 case CFGMECH_2: 268 outb(CONF2_ENABLE_PORT, 0xf0 | (func << 1)); 269 outb(CONF2_FORWARD_PORT, bus); 270 dataport = 0xc000 | (slot << 8) | reg; 271 break; 272 } 273 } 274 return (dataport); 275 } 276 277 /* disable configuration space accesses */ 278 static void 279 pci_cfgdisable(void) 280 { 281 switch (cfgmech) { 282 case CFGMECH_PCIE: 283 case CFGMECH_1: 284 /* 285 * Do nothing for the config mechanism 1 case. 286 * Writing a 0 to the address port can apparently 287 * confuse some bridges and cause spurious 288 * access failures. 289 */ 290 break; 291 case CFGMECH_2: 292 outb(CONF2_ENABLE_PORT, 0); 293 break; 294 } 295 } 296 297 static int 298 pcireg_cfgread(int bus, int slot, int func, int reg, int bytes) 299 { 300 int data = -1; 301 int port; 302 303 mtx_lock_spin(&pcicfg_mtx); 304 port = pci_cfgenable(bus, slot, func, reg, bytes); 305 if (port != 0) { 306 switch (bytes) { 307 case 1: 308 data = inb(port); 309 break; 310 case 2: 311 data = inw(port); 312 break; 313 case 4: 314 data = inl(port); 315 break; 316 } 317 pci_cfgdisable(); 318 } 319 mtx_unlock_spin(&pcicfg_mtx); 320 return (data); 321 } 322 323 static void 324 pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes) 325 { 326 int port; 327 328 mtx_lock_spin(&pcicfg_mtx); 329 port = pci_cfgenable(bus, slot, func, reg, bytes); 330 if (port != 0) { 331 switch (bytes) { 332 case 1: 333 outb(port, data); 334 break; 335 case 2: 336 outw(port, data); 337 break; 338 case 4: 339 outl(port, data); 340 break; 341 } 342 pci_cfgdisable(); 343 } 344 mtx_unlock_spin(&pcicfg_mtx); 345 } 346 347 /* check whether the configuration mechanism has been correctly identified */ 348 static int 349 pci_cfgcheck(int maxdev) 350 { 351 uint32_t id, class; 352 uint8_t header; 353 uint8_t device; 354 int port; 355 356 if (bootverbose) 357 printf("pci_cfgcheck:\tdevice "); 358 359 for (device = 0; device < maxdev; device++) { 360 if (bootverbose) 361 printf("%d ", device); 362 363 port = pci_cfgenable(0, device, 0, 0, 4); 364 id = inl(port); 365 if (id == 0 || id == 0xffffffff) 366 continue; 367 368 port = pci_cfgenable(0, device, 0, 8, 4); 369 class = inl(port) >> 8; 370 if (bootverbose) 371 printf("[class=%06x] ", class); 372 if (class == 0 || (class & 0xf870ff) != 0) 373 continue; 374 375 port = pci_cfgenable(0, device, 0, 14, 1); 376 header = inb(port); 377 if (bootverbose) 378 printf("[hdr=%02x] ", header); 379 if ((header & 0x7e) != 0) 380 continue; 381 382 if (bootverbose) 383 printf("is there (id=%08x)\n", id); 384 385 pci_cfgdisable(); 386 return (1); 387 } 388 if (bootverbose) 389 printf("-- nothing found\n"); 390 391 pci_cfgdisable(); 392 return (0); 393 } 394 395 static int 396 pcireg_cfgopen(void) 397 { 398 uint32_t mode1res, oldval1; 399 uint8_t mode2res, oldval2; 400 401 /* Check for type #1 first. */ 402 oldval1 = inl(CONF1_ADDR_PORT); 403 404 if (bootverbose) { 405 printf("pci_open(1):\tmode 1 addr port (0x0cf8) is 0x%08x\n", 406 oldval1); 407 } 408 409 cfgmech = CFGMECH_1; 410 devmax = 32; 411 412 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK); 413 DELAY(1); 414 mode1res = inl(CONF1_ADDR_PORT); 415 outl(CONF1_ADDR_PORT, oldval1); 416 417 if (bootverbose) 418 printf("pci_open(1a):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 419 CONF1_ENABLE_CHK); 420 421 if (mode1res) { 422 if (pci_cfgcheck(32)) 423 return (cfgmech); 424 } 425 426 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK1); 427 mode1res = inl(CONF1_ADDR_PORT); 428 outl(CONF1_ADDR_PORT, oldval1); 429 430 if (bootverbose) 431 printf("pci_open(1b):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 432 CONF1_ENABLE_CHK1); 433 434 if ((mode1res & CONF1_ENABLE_MSK1) == CONF1_ENABLE_RES1) { 435 if (pci_cfgcheck(32)) 436 return (cfgmech); 437 } 438 439 /* Type #1 didn't work, so try type #2. */ 440 oldval2 = inb(CONF2_ENABLE_PORT); 441 442 if (bootverbose) { 443 printf("pci_open(2):\tmode 2 enable port (0x0cf8) is 0x%02x\n", 444 oldval2); 445 } 446 447 if ((oldval2 & 0xf0) == 0) { 448 449 cfgmech = CFGMECH_2; 450 devmax = 16; 451 452 outb(CONF2_ENABLE_PORT, CONF2_ENABLE_CHK); 453 mode2res = inb(CONF2_ENABLE_PORT); 454 outb(CONF2_ENABLE_PORT, oldval2); 455 456 if (bootverbose) 457 printf("pci_open(2a):\tmode2res=0x%02x (0x%02x)\n", 458 mode2res, CONF2_ENABLE_CHK); 459 460 if (mode2res == CONF2_ENABLE_RES) { 461 if (bootverbose) 462 printf("pci_open(2a):\tnow trying mechanism 2\n"); 463 464 if (pci_cfgcheck(16)) 465 return (cfgmech); 466 } 467 } 468 469 /* Nothing worked, so punt. */ 470 cfgmech = CFGMECH_NONE; 471 devmax = 0; 472 return (cfgmech); 473 } 474 475 int 476 pcie_cfgregopen(uint64_t base, uint8_t minbus, uint8_t maxbus) 477 { 478 struct pcie_cfg_list *pcielist; 479 struct pcie_cfg_elem *pcie_array, *elem; 480 #ifdef SMP 481 struct pcpu *pc; 482 #endif 483 vm_offset_t va; 484 uint32_t val1, val2; 485 int i, slot; 486 487 if (!mcfg_enable) 488 return (0); 489 490 if (minbus != 0) 491 return (0); 492 493 #ifndef PAE 494 if (base >= 0x100000000) { 495 if (bootverbose) 496 printf( 497 "PCI: Memory Mapped PCI configuration area base 0x%jx too high\n", 498 (uintmax_t)base); 499 return (0); 500 } 501 #endif 502 503 if (bootverbose) 504 printf("PCIe: Memory Mapped configuration base @ 0x%jx\n", 505 (uintmax_t)base); 506 507 #ifdef SMP 508 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) 509 #endif 510 { 511 512 pcie_array = malloc(sizeof(struct pcie_cfg_elem) * PCIE_CACHE, 513 M_DEVBUF, M_NOWAIT); 514 if (pcie_array == NULL) 515 return (0); 516 517 va = kva_alloc(PCIE_CACHE * PAGE_SIZE); 518 if (va == 0) { 519 free(pcie_array, M_DEVBUF); 520 return (0); 521 } 522 523 #ifdef SMP 524 pcielist = &pcie_list[pc->pc_cpuid]; 525 #else 526 pcielist = &pcie_list[0]; 527 #endif 528 TAILQ_INIT(pcielist); 529 for (i = 0; i < PCIE_CACHE; i++) { 530 elem = &pcie_array[i]; 531 elem->vapage = va + (i * PAGE_SIZE); 532 elem->papage = 0; 533 TAILQ_INSERT_HEAD(pcielist, elem, elem); 534 } 535 } 536 537 pcie_base = base; 538 pcie_minbus = minbus; 539 pcie_maxbus = maxbus; 540 cfgmech = CFGMECH_PCIE; 541 devmax = 32; 542 543 /* 544 * On some AMD systems, some of the devices on bus 0 are 545 * inaccessible using memory-mapped PCI config access. Walk 546 * bus 0 looking for such devices. For these devices, we will 547 * fall back to using type 1 config access instead. 548 */ 549 if (pci_cfgregopen() != 0) { 550 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 551 val1 = pcireg_cfgread(0, slot, 0, 0, 4); 552 if (val1 == 0xffffffff) 553 continue; 554 555 val2 = pciereg_cfgread(0, slot, 0, 0, 4); 556 if (val2 != val1) 557 pcie_badslots |= (1 << slot); 558 } 559 } 560 561 return (1); 562 } 563 564 #define PCIE_PADDR(base, reg, bus, slot, func) \ 565 ((base) + \ 566 ((((bus) & 0xff) << 20) | \ 567 (((slot) & 0x1f) << 15) | \ 568 (((func) & 0x7) << 12) | \ 569 ((reg) & 0xfff))) 570 571 static __inline vm_offset_t 572 pciereg_findaddr(int bus, unsigned slot, unsigned func, unsigned reg) 573 { 574 struct pcie_cfg_list *pcielist; 575 struct pcie_cfg_elem *elem; 576 vm_paddr_t pa, papage; 577 578 pa = PCIE_PADDR(pcie_base, reg, bus, slot, func); 579 papage = pa & ~PAGE_MASK; 580 581 /* 582 * Find an element in the cache that matches the physical page desired, 583 * or create a new mapping from the least recently used element. 584 * A very simple LRU algorithm is used here, does it need to be more 585 * efficient? 586 */ 587 pcielist = &pcie_list[PCPU_GET(cpuid)]; 588 TAILQ_FOREACH(elem, pcielist, elem) { 589 if (elem->papage == papage) 590 break; 591 } 592 593 if (elem == NULL) { 594 elem = TAILQ_LAST(pcielist, pcie_cfg_list); 595 if (elem->papage != 0) { 596 pmap_kremove(elem->vapage); 597 invlpg(elem->vapage); 598 } 599 pmap_kenter(elem->vapage, papage); 600 elem->papage = papage; 601 } 602 603 if (elem != TAILQ_FIRST(pcielist)) { 604 TAILQ_REMOVE(pcielist, elem, elem); 605 TAILQ_INSERT_HEAD(pcielist, elem, elem); 606 } 607 return (elem->vapage | (pa & PAGE_MASK)); 608 } 609 610 /* 611 * AMD BIOS And Kernel Developer's Guides for CPU families starting with 10h 612 * have a requirement that all accesses to the memory mapped PCI configuration 613 * space are done using AX class of registers. 614 * Since other vendors do not currently have any contradicting requirements 615 * the AMD access pattern is applied universally. 616 */ 617 618 static int 619 pciereg_cfgread(int bus, unsigned slot, unsigned func, unsigned reg, 620 unsigned bytes) 621 { 622 vm_offset_t va; 623 int data = -1; 624 625 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 626 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 627 return (-1); 628 629 critical_enter(); 630 va = pciereg_findaddr(bus, slot, func, reg); 631 632 switch (bytes) { 633 case 4: 634 __asm("movl %1, %0" : "=a" (data) 635 : "m" (*(volatile uint32_t *)va)); 636 break; 637 case 2: 638 __asm("movzwl %1, %0" : "=a" (data) 639 : "m" (*(volatile uint16_t *)va)); 640 break; 641 case 1: 642 __asm("movzbl %1, %0" : "=a" (data) 643 : "m" (*(volatile uint8_t *)va)); 644 break; 645 } 646 647 critical_exit(); 648 return (data); 649 } 650 651 static void 652 pciereg_cfgwrite(int bus, unsigned slot, unsigned func, unsigned reg, int data, 653 unsigned bytes) 654 { 655 vm_offset_t va; 656 657 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 658 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 659 return; 660 661 critical_enter(); 662 va = pciereg_findaddr(bus, slot, func, reg); 663 664 switch (bytes) { 665 case 4: 666 __asm("movl %1, %0" : "=m" (*(volatile uint32_t *)va) 667 : "a" (data)); 668 break; 669 case 2: 670 __asm("movw %1, %0" : "=m" (*(volatile uint16_t *)va) 671 : "a" ((uint16_t)data)); 672 break; 673 case 1: 674 __asm("movb %1, %0" : "=m" (*(volatile uint8_t *)va) 675 : "a" ((uint8_t)data)); 676 break; 677 } 678 679 critical_exit(); 680 } 681