1 /*- 2 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 3 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org> 4 * Copyright (c) 2000, BSDi 5 * Copyright (c) 2004, Scott Long <scottl@freebsd.org> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice unmodified, this list of conditions, and the following 13 * disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/bus.h> 36 #include <sys/lock.h> 37 #include <sys/kernel.h> 38 #include <sys/mutex.h> 39 #include <sys/malloc.h> 40 #include <sys/queue.h> 41 #include <sys/sysctl.h> 42 #include <dev/pci/pcivar.h> 43 #include <dev/pci/pcireg.h> 44 #include <machine/pci_cfgreg.h> 45 #include <machine/pc/bios.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_param.h> 49 #include <vm/vm_kern.h> 50 #include <vm/vm_extern.h> 51 #include <vm/pmap.h> 52 53 #define PRVERB(a) do { \ 54 if (bootverbose) \ 55 printf a ; \ 56 } while(0) 57 58 #define PCIE_CACHE 8 59 struct pcie_cfg_elem { 60 TAILQ_ENTRY(pcie_cfg_elem) elem; 61 vm_offset_t vapage; 62 vm_paddr_t papage; 63 }; 64 65 enum { 66 CFGMECH_NONE = 0, 67 CFGMECH_1, 68 CFGMECH_2, 69 CFGMECH_PCIE, 70 }; 71 72 SYSCTL_DECL(_hw_pci); 73 74 static TAILQ_HEAD(pcie_cfg_list, pcie_cfg_elem) pcie_list[MAXCPU]; 75 static uint64_t pcie_base; 76 static int pcie_minbus, pcie_maxbus; 77 static uint32_t pcie_badslots; 78 static int cfgmech; 79 static int devmax; 80 static struct mtx pcicfg_mtx; 81 static int mcfg_enable = 1; 82 SYSCTL_INT(_hw_pci, OID_AUTO, mcfg, CTLFLAG_RDTUN, &mcfg_enable, 0, 83 "Enable support for PCI-e memory mapped config access"); 84 85 static uint32_t pci_docfgregread(int bus, int slot, int func, int reg, 86 int bytes); 87 static int pcireg_cfgread(int bus, int slot, int func, int reg, int bytes); 88 static void pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes); 89 static int pcireg_cfgopen(void); 90 static int pciereg_cfgread(int bus, unsigned slot, unsigned func, 91 unsigned reg, unsigned bytes); 92 static void pciereg_cfgwrite(int bus, unsigned slot, unsigned func, 93 unsigned reg, int data, unsigned bytes); 94 95 /* 96 * Some BIOS writers seem to want to ignore the spec and put 97 * 0 in the intline rather than 255 to indicate none. Some use 98 * numbers in the range 128-254 to indicate something strange and 99 * apparently undocumented anywhere. Assume these are completely bogus 100 * and map them to 255, which means "none". 101 */ 102 static __inline int 103 pci_i386_map_intline(int line) 104 { 105 if (line == 0 || line >= 128) 106 return (PCI_INVALID_IRQ); 107 return (line); 108 } 109 110 static u_int16_t 111 pcibios_get_version(void) 112 { 113 struct bios_regs args; 114 115 if (PCIbios.ventry == 0) { 116 PRVERB(("pcibios: No call entry point\n")); 117 return (0); 118 } 119 args.eax = PCIBIOS_BIOS_PRESENT; 120 if (bios32(&args, PCIbios.ventry, GSEL(GCODE_SEL, SEL_KPL))) { 121 PRVERB(("pcibios: BIOS_PRESENT call failed\n")); 122 return (0); 123 } 124 if (args.edx != 0x20494350) { 125 PRVERB(("pcibios: BIOS_PRESENT didn't return 'PCI ' in edx\n")); 126 return (0); 127 } 128 return (args.ebx & 0xffff); 129 } 130 131 /* 132 * Initialise access to PCI configuration space 133 */ 134 int 135 pci_cfgregopen(void) 136 { 137 static int opened = 0; 138 uint64_t pciebar; 139 u_int16_t vid, did; 140 u_int16_t v; 141 142 if (opened) 143 return (1); 144 145 if (cfgmech == CFGMECH_NONE && pcireg_cfgopen() == 0) 146 return (0); 147 148 v = pcibios_get_version(); 149 if (v > 0) 150 PRVERB(("pcibios: BIOS version %x.%02x\n", (v & 0xff00) >> 8, 151 v & 0xff)); 152 mtx_init(&pcicfg_mtx, "pcicfg", NULL, MTX_SPIN); 153 opened = 1; 154 155 /* $PIR requires PCI BIOS 2.10 or greater. */ 156 if (v >= 0x0210) 157 pci_pir_open(); 158 159 if (cfgmech == CFGMECH_PCIE) 160 return (1); 161 162 /* 163 * Grope around in the PCI config space to see if this is a 164 * chipset that is capable of doing memory-mapped config cycles. 165 * This also implies that it can do PCIe extended config cycles. 166 */ 167 168 /* Check for supported chipsets */ 169 vid = pci_cfgregread(0, 0, 0, PCIR_VENDOR, 2); 170 did = pci_cfgregread(0, 0, 0, PCIR_DEVICE, 2); 171 switch (vid) { 172 case 0x8086: 173 switch (did) { 174 case 0x3590: 175 case 0x3592: 176 /* Intel 7520 or 7320 */ 177 pciebar = pci_cfgregread(0, 0, 0, 0xce, 2) << 16; 178 pcie_cfgregopen(pciebar, 0, 255); 179 break; 180 case 0x2580: 181 case 0x2584: 182 case 0x2590: 183 /* Intel 915, 925, or 915GM */ 184 pciebar = pci_cfgregread(0, 0, 0, 0x48, 4); 185 pcie_cfgregopen(pciebar, 0, 255); 186 break; 187 } 188 } 189 190 return(1); 191 } 192 193 static uint32_t 194 pci_docfgregread(int bus, int slot, int func, int reg, int bytes) 195 { 196 197 if (cfgmech == CFGMECH_PCIE && 198 (bus >= pcie_minbus && bus <= pcie_maxbus) && 199 (bus != 0 || !(1 << slot & pcie_badslots))) 200 return (pciereg_cfgread(bus, slot, func, reg, bytes)); 201 else 202 return (pcireg_cfgread(bus, slot, func, reg, bytes)); 203 } 204 205 /* 206 * Read configuration space register 207 */ 208 u_int32_t 209 pci_cfgregread(int bus, int slot, int func, int reg, int bytes) 210 { 211 uint32_t line; 212 213 /* 214 * Some BIOS writers seem to want to ignore the spec and put 215 * 0 in the intline rather than 255 to indicate none. The rest of 216 * the code uses 255 as an invalid IRQ. 217 */ 218 if (reg == PCIR_INTLINE && bytes == 1) { 219 line = pci_docfgregread(bus, slot, func, PCIR_INTLINE, 1); 220 return (pci_i386_map_intline(line)); 221 } 222 return (pci_docfgregread(bus, slot, func, reg, bytes)); 223 } 224 225 /* 226 * Write configuration space register 227 */ 228 void 229 pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes) 230 { 231 232 if (cfgmech == CFGMECH_PCIE && 233 (bus >= pcie_minbus && bus <= pcie_maxbus) && 234 (bus != 0 || !(1 << slot & pcie_badslots))) 235 pciereg_cfgwrite(bus, slot, func, reg, data, bytes); 236 else 237 pcireg_cfgwrite(bus, slot, func, reg, data, bytes); 238 } 239 240 /* 241 * Configuration space access using direct register operations 242 */ 243 244 /* enable configuration space accesses and return data port address */ 245 static int 246 pci_cfgenable(unsigned bus, unsigned slot, unsigned func, int reg, int bytes) 247 { 248 int dataport = 0; 249 250 if (bus <= PCI_BUSMAX 251 && slot < devmax 252 && func <= PCI_FUNCMAX 253 && (unsigned)reg <= PCI_REGMAX 254 && bytes != 3 255 && (unsigned)bytes <= 4 256 && (reg & (bytes - 1)) == 0) { 257 switch (cfgmech) { 258 case CFGMECH_PCIE: 259 case CFGMECH_1: 260 outl(CONF1_ADDR_PORT, (1U << 31) 261 | (bus << 16) | (slot << 11) 262 | (func << 8) | (reg & ~0x03)); 263 dataport = CONF1_DATA_PORT + (reg & 0x03); 264 break; 265 case CFGMECH_2: 266 outb(CONF2_ENABLE_PORT, 0xf0 | (func << 1)); 267 outb(CONF2_FORWARD_PORT, bus); 268 dataport = 0xc000 | (slot << 8) | reg; 269 break; 270 } 271 } 272 return (dataport); 273 } 274 275 /* disable configuration space accesses */ 276 static void 277 pci_cfgdisable(void) 278 { 279 switch (cfgmech) { 280 case CFGMECH_PCIE: 281 case CFGMECH_1: 282 /* 283 * Do nothing for the config mechanism 1 case. 284 * Writing a 0 to the address port can apparently 285 * confuse some bridges and cause spurious 286 * access failures. 287 */ 288 break; 289 case CFGMECH_2: 290 outb(CONF2_ENABLE_PORT, 0); 291 break; 292 } 293 } 294 295 static int 296 pcireg_cfgread(int bus, int slot, int func, int reg, int bytes) 297 { 298 int data = -1; 299 int port; 300 301 mtx_lock_spin(&pcicfg_mtx); 302 port = pci_cfgenable(bus, slot, func, reg, bytes); 303 if (port != 0) { 304 switch (bytes) { 305 case 1: 306 data = inb(port); 307 break; 308 case 2: 309 data = inw(port); 310 break; 311 case 4: 312 data = inl(port); 313 break; 314 } 315 pci_cfgdisable(); 316 } 317 mtx_unlock_spin(&pcicfg_mtx); 318 return (data); 319 } 320 321 static void 322 pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes) 323 { 324 int port; 325 326 mtx_lock_spin(&pcicfg_mtx); 327 port = pci_cfgenable(bus, slot, func, reg, bytes); 328 if (port != 0) { 329 switch (bytes) { 330 case 1: 331 outb(port, data); 332 break; 333 case 2: 334 outw(port, data); 335 break; 336 case 4: 337 outl(port, data); 338 break; 339 } 340 pci_cfgdisable(); 341 } 342 mtx_unlock_spin(&pcicfg_mtx); 343 } 344 345 /* check whether the configuration mechanism has been correctly identified */ 346 static int 347 pci_cfgcheck(int maxdev) 348 { 349 uint32_t id, class; 350 uint8_t header; 351 uint8_t device; 352 int port; 353 354 if (bootverbose) 355 printf("pci_cfgcheck:\tdevice "); 356 357 for (device = 0; device < maxdev; device++) { 358 if (bootverbose) 359 printf("%d ", device); 360 361 port = pci_cfgenable(0, device, 0, 0, 4); 362 id = inl(port); 363 if (id == 0 || id == 0xffffffff) 364 continue; 365 366 port = pci_cfgenable(0, device, 0, 8, 4); 367 class = inl(port) >> 8; 368 if (bootverbose) 369 printf("[class=%06x] ", class); 370 if (class == 0 || (class & 0xf870ff) != 0) 371 continue; 372 373 port = pci_cfgenable(0, device, 0, 14, 1); 374 header = inb(port); 375 if (bootverbose) 376 printf("[hdr=%02x] ", header); 377 if ((header & 0x7e) != 0) 378 continue; 379 380 if (bootverbose) 381 printf("is there (id=%08x)\n", id); 382 383 pci_cfgdisable(); 384 return (1); 385 } 386 if (bootverbose) 387 printf("-- nothing found\n"); 388 389 pci_cfgdisable(); 390 return (0); 391 } 392 393 static int 394 pcireg_cfgopen(void) 395 { 396 uint32_t mode1res, oldval1; 397 uint8_t mode2res, oldval2; 398 399 /* Check for type #1 first. */ 400 oldval1 = inl(CONF1_ADDR_PORT); 401 402 if (bootverbose) { 403 printf("pci_open(1):\tmode 1 addr port (0x0cf8) is 0x%08x\n", 404 oldval1); 405 } 406 407 cfgmech = CFGMECH_1; 408 devmax = 32; 409 410 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK); 411 DELAY(1); 412 mode1res = inl(CONF1_ADDR_PORT); 413 outl(CONF1_ADDR_PORT, oldval1); 414 415 if (bootverbose) 416 printf("pci_open(1a):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 417 CONF1_ENABLE_CHK); 418 419 if (mode1res) { 420 if (pci_cfgcheck(32)) 421 return (cfgmech); 422 } 423 424 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK1); 425 mode1res = inl(CONF1_ADDR_PORT); 426 outl(CONF1_ADDR_PORT, oldval1); 427 428 if (bootverbose) 429 printf("pci_open(1b):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 430 CONF1_ENABLE_CHK1); 431 432 if ((mode1res & CONF1_ENABLE_MSK1) == CONF1_ENABLE_RES1) { 433 if (pci_cfgcheck(32)) 434 return (cfgmech); 435 } 436 437 /* Type #1 didn't work, so try type #2. */ 438 oldval2 = inb(CONF2_ENABLE_PORT); 439 440 if (bootverbose) { 441 printf("pci_open(2):\tmode 2 enable port (0x0cf8) is 0x%02x\n", 442 oldval2); 443 } 444 445 if ((oldval2 & 0xf0) == 0) { 446 447 cfgmech = CFGMECH_2; 448 devmax = 16; 449 450 outb(CONF2_ENABLE_PORT, CONF2_ENABLE_CHK); 451 mode2res = inb(CONF2_ENABLE_PORT); 452 outb(CONF2_ENABLE_PORT, oldval2); 453 454 if (bootverbose) 455 printf("pci_open(2a):\tmode2res=0x%02x (0x%02x)\n", 456 mode2res, CONF2_ENABLE_CHK); 457 458 if (mode2res == CONF2_ENABLE_RES) { 459 if (bootverbose) 460 printf("pci_open(2a):\tnow trying mechanism 2\n"); 461 462 if (pci_cfgcheck(16)) 463 return (cfgmech); 464 } 465 } 466 467 /* Nothing worked, so punt. */ 468 cfgmech = CFGMECH_NONE; 469 devmax = 0; 470 return (cfgmech); 471 } 472 473 int 474 pcie_cfgregopen(uint64_t base, uint8_t minbus, uint8_t maxbus) 475 { 476 struct pcie_cfg_list *pcielist; 477 struct pcie_cfg_elem *pcie_array, *elem; 478 #ifdef SMP 479 struct pcpu *pc; 480 #endif 481 vm_offset_t va; 482 uint32_t val1, val2; 483 int i, slot; 484 485 if (!mcfg_enable) 486 return (0); 487 488 if (minbus != 0) 489 return (0); 490 491 #ifndef PAE 492 if (base >= 0x100000000) { 493 if (bootverbose) 494 printf( 495 "PCI: Memory Mapped PCI configuration area base 0x%jx too high\n", 496 (uintmax_t)base); 497 return (0); 498 } 499 #endif 500 501 if (bootverbose) 502 printf("PCIe: Memory Mapped configuration base @ 0x%jx\n", 503 (uintmax_t)base); 504 505 #ifdef SMP 506 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) 507 #endif 508 { 509 510 pcie_array = malloc(sizeof(struct pcie_cfg_elem) * PCIE_CACHE, 511 M_DEVBUF, M_NOWAIT); 512 if (pcie_array == NULL) 513 return (0); 514 515 va = kva_alloc(PCIE_CACHE * PAGE_SIZE); 516 if (va == 0) { 517 free(pcie_array, M_DEVBUF); 518 return (0); 519 } 520 521 #ifdef SMP 522 pcielist = &pcie_list[pc->pc_cpuid]; 523 #else 524 pcielist = &pcie_list[0]; 525 #endif 526 TAILQ_INIT(pcielist); 527 for (i = 0; i < PCIE_CACHE; i++) { 528 elem = &pcie_array[i]; 529 elem->vapage = va + (i * PAGE_SIZE); 530 elem->papage = 0; 531 TAILQ_INSERT_HEAD(pcielist, elem, elem); 532 } 533 } 534 535 pcie_base = base; 536 pcie_minbus = minbus; 537 pcie_maxbus = maxbus; 538 cfgmech = CFGMECH_PCIE; 539 devmax = 32; 540 541 /* 542 * On some AMD systems, some of the devices on bus 0 are 543 * inaccessible using memory-mapped PCI config access. Walk 544 * bus 0 looking for such devices. For these devices, we will 545 * fall back to using type 1 config access instead. 546 */ 547 if (pci_cfgregopen() != 0) { 548 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 549 val1 = pcireg_cfgread(0, slot, 0, 0, 4); 550 if (val1 == 0xffffffff) 551 continue; 552 553 val2 = pciereg_cfgread(0, slot, 0, 0, 4); 554 if (val2 != val1) 555 pcie_badslots |= (1 << slot); 556 } 557 } 558 559 return (1); 560 } 561 562 #define PCIE_PADDR(base, reg, bus, slot, func) \ 563 ((base) + \ 564 ((((bus) & 0xff) << 20) | \ 565 (((slot) & 0x1f) << 15) | \ 566 (((func) & 0x7) << 12) | \ 567 ((reg) & 0xfff))) 568 569 static __inline vm_offset_t 570 pciereg_findaddr(int bus, unsigned slot, unsigned func, unsigned reg) 571 { 572 struct pcie_cfg_list *pcielist; 573 struct pcie_cfg_elem *elem; 574 vm_paddr_t pa, papage; 575 576 pa = PCIE_PADDR(pcie_base, reg, bus, slot, func); 577 papage = pa & ~PAGE_MASK; 578 579 /* 580 * Find an element in the cache that matches the physical page desired, 581 * or create a new mapping from the least recently used element. 582 * A very simple LRU algorithm is used here, does it need to be more 583 * efficient? 584 */ 585 pcielist = &pcie_list[PCPU_GET(cpuid)]; 586 TAILQ_FOREACH(elem, pcielist, elem) { 587 if (elem->papage == papage) 588 break; 589 } 590 591 if (elem == NULL) { 592 elem = TAILQ_LAST(pcielist, pcie_cfg_list); 593 if (elem->papage != 0) { 594 pmap_kremove(elem->vapage); 595 invlpg(elem->vapage); 596 } 597 pmap_kenter(elem->vapage, papage); 598 elem->papage = papage; 599 } 600 601 if (elem != TAILQ_FIRST(pcielist)) { 602 TAILQ_REMOVE(pcielist, elem, elem); 603 TAILQ_INSERT_HEAD(pcielist, elem, elem); 604 } 605 return (elem->vapage | (pa & PAGE_MASK)); 606 } 607 608 /* 609 * AMD BIOS And Kernel Developer's Guides for CPU families starting with 10h 610 * have a requirement that all accesses to the memory mapped PCI configuration 611 * space are done using AX class of registers. 612 * Since other vendors do not currently have any contradicting requirements 613 * the AMD access pattern is applied universally. 614 */ 615 616 static int 617 pciereg_cfgread(int bus, unsigned slot, unsigned func, unsigned reg, 618 unsigned bytes) 619 { 620 vm_offset_t va; 621 int data = -1; 622 623 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 624 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 625 return (-1); 626 627 critical_enter(); 628 va = pciereg_findaddr(bus, slot, func, reg); 629 630 switch (bytes) { 631 case 4: 632 __asm("movl %1, %0" : "=a" (data) 633 : "m" (*(volatile uint32_t *)va)); 634 break; 635 case 2: 636 __asm("movzwl %1, %0" : "=a" (data) 637 : "m" (*(volatile uint16_t *)va)); 638 break; 639 case 1: 640 __asm("movzbl %1, %0" : "=a" (data) 641 : "m" (*(volatile uint8_t *)va)); 642 break; 643 } 644 645 critical_exit(); 646 return (data); 647 } 648 649 static void 650 pciereg_cfgwrite(int bus, unsigned slot, unsigned func, unsigned reg, int data, 651 unsigned bytes) 652 { 653 vm_offset_t va; 654 655 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 656 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 657 return; 658 659 critical_enter(); 660 va = pciereg_findaddr(bus, slot, func, reg); 661 662 switch (bytes) { 663 case 4: 664 __asm("movl %1, %0" : "=m" (*(volatile uint32_t *)va) 665 : "a" (data)); 666 break; 667 case 2: 668 __asm("movw %1, %0" : "=m" (*(volatile uint16_t *)va) 669 : "a" ((uint16_t)data)); 670 break; 671 case 1: 672 __asm("movb %1, %0" : "=m" (*(volatile uint8_t *)va) 673 : "a" ((uint8_t)data)); 674 break; 675 } 676 677 critical_exit(); 678 } 679