1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org> 6 * Copyright (c) 2000, BSDi 7 * Copyright (c) 2004, Scott Long <scottl@freebsd.org> 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/bus.h> 38 #include <sys/lock.h> 39 #include <sys/kernel.h> 40 #include <sys/mutex.h> 41 #include <sys/malloc.h> 42 #include <sys/queue.h> 43 #include <sys/sysctl.h> 44 #include <dev/pci/pcivar.h> 45 #include <dev/pci/pcireg.h> 46 #include <machine/pci_cfgreg.h> 47 #include <machine/pc/bios.h> 48 49 #include <vm/vm.h> 50 #include <vm/vm_param.h> 51 #include <vm/vm_kern.h> 52 #include <vm/vm_extern.h> 53 #include <vm/pmap.h> 54 55 #define PRVERB(a) do { \ 56 if (bootverbose) \ 57 printf a ; \ 58 } while(0) 59 60 #define PCIE_CACHE 8 61 struct pcie_cfg_elem { 62 TAILQ_ENTRY(pcie_cfg_elem) elem; 63 vm_offset_t vapage; 64 vm_paddr_t papage; 65 }; 66 67 SYSCTL_DECL(_hw_pci); 68 69 static TAILQ_HEAD(pcie_cfg_list, pcie_cfg_elem) pcie_list[MAXCPU]; 70 static uint64_t pcie_base; 71 static int pcie_minbus, pcie_maxbus; 72 static uint32_t pcie_badslots; 73 int cfgmech; 74 static int devmax; 75 static struct mtx pcicfg_mtx; 76 static int mcfg_enable = 1; 77 SYSCTL_INT(_hw_pci, OID_AUTO, mcfg, CTLFLAG_RDTUN, &mcfg_enable, 0, 78 "Enable support for PCI-e memory mapped config access"); 79 80 static uint32_t pci_docfgregread(int bus, int slot, int func, int reg, 81 int bytes); 82 static int pcireg_cfgread(int bus, int slot, int func, int reg, int bytes); 83 static void pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes); 84 static int pcireg_cfgopen(void); 85 static int pciereg_cfgread(int bus, unsigned slot, unsigned func, 86 unsigned reg, unsigned bytes); 87 static void pciereg_cfgwrite(int bus, unsigned slot, unsigned func, 88 unsigned reg, int data, unsigned bytes); 89 90 /* 91 * Some BIOS writers seem to want to ignore the spec and put 92 * 0 in the intline rather than 255 to indicate none. Some use 93 * numbers in the range 128-254 to indicate something strange and 94 * apparently undocumented anywhere. Assume these are completely bogus 95 * and map them to 255, which means "none". 96 */ 97 static __inline int 98 pci_i386_map_intline(int line) 99 { 100 if (line == 0 || line >= 128) 101 return (PCI_INVALID_IRQ); 102 return (line); 103 } 104 105 static u_int16_t 106 pcibios_get_version(void) 107 { 108 struct bios_regs args; 109 110 if (PCIbios.ventry == 0) { 111 PRVERB(("pcibios: No call entry point\n")); 112 return (0); 113 } 114 args.eax = PCIBIOS_BIOS_PRESENT; 115 if (bios32(&args, PCIbios.ventry, GSEL(GCODE_SEL, SEL_KPL))) { 116 PRVERB(("pcibios: BIOS_PRESENT call failed\n")); 117 return (0); 118 } 119 if (args.edx != 0x20494350) { 120 PRVERB(("pcibios: BIOS_PRESENT didn't return 'PCI ' in edx\n")); 121 return (0); 122 } 123 return (args.ebx & 0xffff); 124 } 125 126 /* 127 * Initialise access to PCI configuration space 128 */ 129 int 130 pci_cfgregopen(void) 131 { 132 uint16_t v; 133 static int opened = 0; 134 135 if (opened) 136 return (1); 137 138 if (cfgmech == CFGMECH_NONE && pcireg_cfgopen() == 0) 139 return (0); 140 141 v = pcibios_get_version(); 142 if (v > 0) 143 PRVERB(("pcibios: BIOS version %x.%02x\n", (v & 0xff00) >> 8, 144 v & 0xff)); 145 mtx_init(&pcicfg_mtx, "pcicfg", NULL, MTX_SPIN); 146 opened = 1; 147 148 /* $PIR requires PCI BIOS 2.10 or greater. */ 149 if (v >= 0x0210) 150 pci_pir_open(); 151 152 return (1); 153 } 154 155 static uint32_t 156 pci_docfgregread(int bus, int slot, int func, int reg, int bytes) 157 { 158 159 if (cfgmech == CFGMECH_PCIE && 160 (bus >= pcie_minbus && bus <= pcie_maxbus) && 161 (bus != 0 || !(1 << slot & pcie_badslots))) 162 return (pciereg_cfgread(bus, slot, func, reg, bytes)); 163 else 164 return (pcireg_cfgread(bus, slot, func, reg, bytes)); 165 } 166 167 /* 168 * Read configuration space register 169 */ 170 u_int32_t 171 pci_cfgregread(int bus, int slot, int func, int reg, int bytes) 172 { 173 uint32_t line; 174 175 /* 176 * Some BIOS writers seem to want to ignore the spec and put 177 * 0 in the intline rather than 255 to indicate none. The rest of 178 * the code uses 255 as an invalid IRQ. 179 */ 180 if (reg == PCIR_INTLINE && bytes == 1) { 181 line = pci_docfgregread(bus, slot, func, PCIR_INTLINE, 1); 182 return (pci_i386_map_intline(line)); 183 } 184 return (pci_docfgregread(bus, slot, func, reg, bytes)); 185 } 186 187 /* 188 * Write configuration space register 189 */ 190 void 191 pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes) 192 { 193 194 if (cfgmech == CFGMECH_PCIE && 195 (bus >= pcie_minbus && bus <= pcie_maxbus) && 196 (bus != 0 || !(1 << slot & pcie_badslots))) 197 pciereg_cfgwrite(bus, slot, func, reg, data, bytes); 198 else 199 pcireg_cfgwrite(bus, slot, func, reg, data, bytes); 200 } 201 202 /* 203 * Configuration space access using direct register operations 204 */ 205 206 /* enable configuration space accesses and return data port address */ 207 static int 208 pci_cfgenable(unsigned bus, unsigned slot, unsigned func, int reg, int bytes) 209 { 210 int dataport = 0; 211 212 if (bus <= PCI_BUSMAX 213 && slot < devmax 214 && func <= PCI_FUNCMAX 215 && (unsigned)reg <= PCI_REGMAX 216 && bytes != 3 217 && (unsigned)bytes <= 4 218 && (reg & (bytes - 1)) == 0) { 219 switch (cfgmech) { 220 case CFGMECH_PCIE: 221 case CFGMECH_1: 222 outl(CONF1_ADDR_PORT, (1U << 31) 223 | (bus << 16) | (slot << 11) 224 | (func << 8) | (reg & ~0x03)); 225 dataport = CONF1_DATA_PORT + (reg & 0x03); 226 break; 227 case CFGMECH_2: 228 outb(CONF2_ENABLE_PORT, 0xf0 | (func << 1)); 229 outb(CONF2_FORWARD_PORT, bus); 230 dataport = 0xc000 | (slot << 8) | reg; 231 break; 232 } 233 } 234 return (dataport); 235 } 236 237 /* disable configuration space accesses */ 238 static void 239 pci_cfgdisable(void) 240 { 241 switch (cfgmech) { 242 case CFGMECH_PCIE: 243 case CFGMECH_1: 244 /* 245 * Do nothing for the config mechanism 1 case. 246 * Writing a 0 to the address port can apparently 247 * confuse some bridges and cause spurious 248 * access failures. 249 */ 250 break; 251 case CFGMECH_2: 252 outb(CONF2_ENABLE_PORT, 0); 253 break; 254 } 255 } 256 257 static int 258 pcireg_cfgread(int bus, int slot, int func, int reg, int bytes) 259 { 260 int data = -1; 261 int port; 262 263 mtx_lock_spin(&pcicfg_mtx); 264 port = pci_cfgenable(bus, slot, func, reg, bytes); 265 if (port != 0) { 266 switch (bytes) { 267 case 1: 268 data = inb(port); 269 break; 270 case 2: 271 data = inw(port); 272 break; 273 case 4: 274 data = inl(port); 275 break; 276 } 277 pci_cfgdisable(); 278 } 279 mtx_unlock_spin(&pcicfg_mtx); 280 return (data); 281 } 282 283 static void 284 pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes) 285 { 286 int port; 287 288 mtx_lock_spin(&pcicfg_mtx); 289 port = pci_cfgenable(bus, slot, func, reg, bytes); 290 if (port != 0) { 291 switch (bytes) { 292 case 1: 293 outb(port, data); 294 break; 295 case 2: 296 outw(port, data); 297 break; 298 case 4: 299 outl(port, data); 300 break; 301 } 302 pci_cfgdisable(); 303 } 304 mtx_unlock_spin(&pcicfg_mtx); 305 } 306 307 /* check whether the configuration mechanism has been correctly identified */ 308 static int 309 pci_cfgcheck(int maxdev) 310 { 311 uint32_t id, class; 312 uint8_t header; 313 uint8_t device; 314 int port; 315 316 if (bootverbose) 317 printf("pci_cfgcheck:\tdevice "); 318 319 for (device = 0; device < maxdev; device++) { 320 if (bootverbose) 321 printf("%d ", device); 322 323 port = pci_cfgenable(0, device, 0, 0, 4); 324 id = inl(port); 325 if (id == 0 || id == 0xffffffff) 326 continue; 327 328 port = pci_cfgenable(0, device, 0, 8, 4); 329 class = inl(port) >> 8; 330 if (bootverbose) 331 printf("[class=%06x] ", class); 332 if (class == 0 || (class & 0xf870ff) != 0) 333 continue; 334 335 port = pci_cfgenable(0, device, 0, 14, 1); 336 header = inb(port); 337 if (bootverbose) 338 printf("[hdr=%02x] ", header); 339 if ((header & 0x7e) != 0) 340 continue; 341 342 if (bootverbose) 343 printf("is there (id=%08x)\n", id); 344 345 pci_cfgdisable(); 346 return (1); 347 } 348 if (bootverbose) 349 printf("-- nothing found\n"); 350 351 pci_cfgdisable(); 352 return (0); 353 } 354 355 static int 356 pcireg_cfgopen(void) 357 { 358 uint32_t mode1res, oldval1; 359 uint8_t mode2res, oldval2; 360 361 /* Check for type #1 first. */ 362 oldval1 = inl(CONF1_ADDR_PORT); 363 364 if (bootverbose) { 365 printf("pci_open(1):\tmode 1 addr port (0x0cf8) is 0x%08x\n", 366 oldval1); 367 } 368 369 cfgmech = CFGMECH_1; 370 devmax = 32; 371 372 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK); 373 DELAY(1); 374 mode1res = inl(CONF1_ADDR_PORT); 375 outl(CONF1_ADDR_PORT, oldval1); 376 377 if (bootverbose) 378 printf("pci_open(1a):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 379 CONF1_ENABLE_CHK); 380 381 if (mode1res) { 382 if (pci_cfgcheck(32)) 383 return (cfgmech); 384 } 385 386 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK1); 387 mode1res = inl(CONF1_ADDR_PORT); 388 outl(CONF1_ADDR_PORT, oldval1); 389 390 if (bootverbose) 391 printf("pci_open(1b):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 392 CONF1_ENABLE_CHK1); 393 394 if ((mode1res & CONF1_ENABLE_MSK1) == CONF1_ENABLE_RES1) { 395 if (pci_cfgcheck(32)) 396 return (cfgmech); 397 } 398 399 /* Type #1 didn't work, so try type #2. */ 400 oldval2 = inb(CONF2_ENABLE_PORT); 401 402 if (bootverbose) { 403 printf("pci_open(2):\tmode 2 enable port (0x0cf8) is 0x%02x\n", 404 oldval2); 405 } 406 407 if ((oldval2 & 0xf0) == 0) { 408 cfgmech = CFGMECH_2; 409 devmax = 16; 410 411 outb(CONF2_ENABLE_PORT, CONF2_ENABLE_CHK); 412 mode2res = inb(CONF2_ENABLE_PORT); 413 outb(CONF2_ENABLE_PORT, oldval2); 414 415 if (bootverbose) 416 printf("pci_open(2a):\tmode2res=0x%02x (0x%02x)\n", 417 mode2res, CONF2_ENABLE_CHK); 418 419 if (mode2res == CONF2_ENABLE_RES) { 420 if (bootverbose) 421 printf("pci_open(2a):\tnow trying mechanism 2\n"); 422 423 if (pci_cfgcheck(16)) 424 return (cfgmech); 425 } 426 } 427 428 /* Nothing worked, so punt. */ 429 cfgmech = CFGMECH_NONE; 430 devmax = 0; 431 return (cfgmech); 432 } 433 434 int 435 pcie_cfgregopen(uint64_t base, uint8_t minbus, uint8_t maxbus) 436 { 437 struct pcie_cfg_list *pcielist; 438 struct pcie_cfg_elem *pcie_array, *elem; 439 #ifdef SMP 440 struct pcpu *pc; 441 #endif 442 vm_offset_t va; 443 uint32_t val1, val2; 444 int i, slot; 445 446 if (!mcfg_enable) 447 return (0); 448 449 if (minbus != 0) 450 return (0); 451 452 if (!pae_mode && base >= 0x100000000) { 453 if (bootverbose) 454 printf( 455 "PCI: Memory Mapped PCI configuration area base 0x%jx too high\n", 456 (uintmax_t)base); 457 return (0); 458 } 459 460 if (bootverbose) 461 printf("PCIe: Memory Mapped configuration base @ 0x%jx\n", 462 (uintmax_t)base); 463 464 #ifdef SMP 465 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) 466 #endif 467 { 468 pcie_array = malloc(sizeof(struct pcie_cfg_elem) * PCIE_CACHE, 469 M_DEVBUF, M_NOWAIT); 470 if (pcie_array == NULL) 471 return (0); 472 473 va = kva_alloc(PCIE_CACHE * PAGE_SIZE); 474 if (va == 0) { 475 free(pcie_array, M_DEVBUF); 476 return (0); 477 } 478 479 #ifdef SMP 480 pcielist = &pcie_list[pc->pc_cpuid]; 481 #else 482 pcielist = &pcie_list[0]; 483 #endif 484 TAILQ_INIT(pcielist); 485 for (i = 0; i < PCIE_CACHE; i++) { 486 elem = &pcie_array[i]; 487 elem->vapage = va + (i * PAGE_SIZE); 488 elem->papage = 0; 489 TAILQ_INSERT_HEAD(pcielist, elem, elem); 490 } 491 } 492 493 pcie_base = base; 494 pcie_minbus = minbus; 495 pcie_maxbus = maxbus; 496 cfgmech = CFGMECH_PCIE; 497 devmax = 32; 498 499 /* 500 * On some AMD systems, some of the devices on bus 0 are 501 * inaccessible using memory-mapped PCI config access. Walk 502 * bus 0 looking for such devices. For these devices, we will 503 * fall back to using type 1 config access instead. 504 */ 505 if (pci_cfgregopen() != 0) { 506 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 507 val1 = pcireg_cfgread(0, slot, 0, 0, 4); 508 if (val1 == 0xffffffff) 509 continue; 510 511 val2 = pciereg_cfgread(0, slot, 0, 0, 4); 512 if (val2 != val1) 513 pcie_badslots |= (1 << slot); 514 } 515 } 516 517 return (1); 518 } 519 520 #define PCIE_PADDR(base, reg, bus, slot, func) \ 521 ((base) + \ 522 ((((bus) & 0xff) << 20) | \ 523 (((slot) & 0x1f) << 15) | \ 524 (((func) & 0x7) << 12) | \ 525 ((reg) & 0xfff))) 526 527 static __inline vm_offset_t 528 pciereg_findaddr(int bus, unsigned slot, unsigned func, unsigned reg) 529 { 530 struct pcie_cfg_list *pcielist; 531 struct pcie_cfg_elem *elem; 532 vm_paddr_t pa, papage; 533 534 pa = PCIE_PADDR(pcie_base, reg, bus, slot, func); 535 papage = pa & ~PAGE_MASK; 536 537 /* 538 * Find an element in the cache that matches the physical page desired, 539 * or create a new mapping from the least recently used element. 540 * A very simple LRU algorithm is used here, does it need to be more 541 * efficient? 542 */ 543 pcielist = &pcie_list[PCPU_GET(cpuid)]; 544 TAILQ_FOREACH(elem, pcielist, elem) { 545 if (elem->papage == papage) 546 break; 547 } 548 549 if (elem == NULL) { 550 elem = TAILQ_LAST(pcielist, pcie_cfg_list); 551 if (elem->papage != 0) { 552 pmap_kremove(elem->vapage); 553 invlpg(elem->vapage); 554 } 555 pmap_kenter(elem->vapage, papage); 556 elem->papage = papage; 557 } 558 559 if (elem != TAILQ_FIRST(pcielist)) { 560 TAILQ_REMOVE(pcielist, elem, elem); 561 TAILQ_INSERT_HEAD(pcielist, elem, elem); 562 } 563 return (elem->vapage | (pa & PAGE_MASK)); 564 } 565 566 /* 567 * AMD BIOS And Kernel Developer's Guides for CPU families starting with 10h 568 * have a requirement that all accesses to the memory mapped PCI configuration 569 * space are done using AX class of registers. 570 * Since other vendors do not currently have any contradicting requirements 571 * the AMD access pattern is applied universally. 572 */ 573 574 static int 575 pciereg_cfgread(int bus, unsigned slot, unsigned func, unsigned reg, 576 unsigned bytes) 577 { 578 vm_offset_t va; 579 int data = -1; 580 581 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 582 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 583 return (-1); 584 585 critical_enter(); 586 va = pciereg_findaddr(bus, slot, func, reg); 587 588 switch (bytes) { 589 case 4: 590 __asm("movl %1, %0" : "=a" (data) 591 : "m" (*(volatile uint32_t *)va)); 592 break; 593 case 2: 594 __asm("movzwl %1, %0" : "=a" (data) 595 : "m" (*(volatile uint16_t *)va)); 596 break; 597 case 1: 598 __asm("movzbl %1, %0" : "=a" (data) 599 : "m" (*(volatile uint8_t *)va)); 600 break; 601 } 602 603 critical_exit(); 604 return (data); 605 } 606 607 static void 608 pciereg_cfgwrite(int bus, unsigned slot, unsigned func, unsigned reg, int data, 609 unsigned bytes) 610 { 611 vm_offset_t va; 612 613 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 614 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 615 return; 616 617 critical_enter(); 618 va = pciereg_findaddr(bus, slot, func, reg); 619 620 switch (bytes) { 621 case 4: 622 __asm("movl %1, %0" : "=m" (*(volatile uint32_t *)va) 623 : "a" (data)); 624 break; 625 case 2: 626 __asm("movw %1, %0" : "=m" (*(volatile uint16_t *)va) 627 : "a" ((uint16_t)data)); 628 break; 629 case 1: 630 __asm("movb %1, %0" : "=m" (*(volatile uint8_t *)va) 631 : "a" ((uint8_t)data)); 632 break; 633 } 634 635 critical_exit(); 636 } 637