1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org> 6 * Copyright (c) 2000, BSDi 7 * Copyright (c) 2004, Scott Long <scottl@freebsd.org> 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/bus.h> 36 #include <sys/lock.h> 37 #include <sys/kernel.h> 38 #include <sys/mutex.h> 39 #include <sys/malloc.h> 40 #include <sys/queue.h> 41 #include <sys/sysctl.h> 42 #include <dev/pci/pcivar.h> 43 #include <dev/pci/pcireg.h> 44 #include <machine/pci_cfgreg.h> 45 #include <machine/pc/bios.h> 46 47 #include <vm/vm.h> 48 #include <vm/vm_param.h> 49 #include <vm/vm_kern.h> 50 #include <vm/vm_extern.h> 51 #include <vm/pmap.h> 52 53 #define PRVERB(a) do { \ 54 if (bootverbose) \ 55 printf a ; \ 56 } while(0) 57 58 #define PCIE_CACHE 8 59 struct pcie_cfg_elem { 60 TAILQ_ENTRY(pcie_cfg_elem) elem; 61 vm_offset_t vapage; 62 vm_paddr_t papage; 63 }; 64 65 SYSCTL_DECL(_hw_pci); 66 67 static TAILQ_HEAD(pcie_cfg_list, pcie_cfg_elem) pcie_list[MAXCPU]; 68 static uint64_t pcie_base; 69 static int pcie_minbus, pcie_maxbus; 70 static uint32_t pcie_badslots; 71 int cfgmech; 72 static int devmax; 73 static struct mtx pcicfg_mtx; 74 static int mcfg_enable = 1; 75 SYSCTL_INT(_hw_pci, OID_AUTO, mcfg, CTLFLAG_RDTUN, &mcfg_enable, 0, 76 "Enable support for PCI-e memory mapped config access"); 77 78 static uint32_t pci_docfgregread(int bus, int slot, int func, int reg, 79 int bytes); 80 static int pcireg_cfgread(int bus, int slot, int func, int reg, int bytes); 81 static void pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes); 82 static int pcireg_cfgopen(void); 83 static int pciereg_cfgread(int bus, unsigned slot, unsigned func, 84 unsigned reg, unsigned bytes); 85 static void pciereg_cfgwrite(int bus, unsigned slot, unsigned func, 86 unsigned reg, int data, unsigned bytes); 87 88 /* 89 * Some BIOS writers seem to want to ignore the spec and put 90 * 0 in the intline rather than 255 to indicate none. Some use 91 * numbers in the range 128-254 to indicate something strange and 92 * apparently undocumented anywhere. Assume these are completely bogus 93 * and map them to 255, which means "none". 94 */ 95 static __inline int 96 pci_i386_map_intline(int line) 97 { 98 if (line == 0 || line >= 128) 99 return (PCI_INVALID_IRQ); 100 return (line); 101 } 102 103 static u_int16_t 104 pcibios_get_version(void) 105 { 106 struct bios_regs args; 107 108 if (PCIbios.ventry == 0) { 109 PRVERB(("pcibios: No call entry point\n")); 110 return (0); 111 } 112 args.eax = PCIBIOS_BIOS_PRESENT; 113 if (bios32(&args, PCIbios.ventry, GSEL(GCODE_SEL, SEL_KPL))) { 114 PRVERB(("pcibios: BIOS_PRESENT call failed\n")); 115 return (0); 116 } 117 if (args.edx != 0x20494350) { 118 PRVERB(("pcibios: BIOS_PRESENT didn't return 'PCI ' in edx\n")); 119 return (0); 120 } 121 return (args.ebx & 0xffff); 122 } 123 124 /* 125 * Initialise access to PCI configuration space 126 */ 127 int 128 pci_cfgregopen(void) 129 { 130 uint16_t v; 131 static int opened = 0; 132 133 if (opened) 134 return (1); 135 136 if (cfgmech == CFGMECH_NONE && pcireg_cfgopen() == 0) 137 return (0); 138 139 v = pcibios_get_version(); 140 if (v > 0) 141 PRVERB(("pcibios: BIOS version %x.%02x\n", (v & 0xff00) >> 8, 142 v & 0xff)); 143 mtx_init(&pcicfg_mtx, "pcicfg", NULL, MTX_SPIN); 144 opened = 1; 145 146 /* $PIR requires PCI BIOS 2.10 or greater. */ 147 if (v >= 0x0210) 148 pci_pir_open(); 149 150 return (1); 151 } 152 153 static uint32_t 154 pci_docfgregread(int bus, int slot, int func, int reg, int bytes) 155 { 156 157 if (cfgmech == CFGMECH_PCIE && 158 (bus >= pcie_minbus && bus <= pcie_maxbus) && 159 (bus != 0 || !(1 << slot & pcie_badslots))) 160 return (pciereg_cfgread(bus, slot, func, reg, bytes)); 161 else 162 return (pcireg_cfgread(bus, slot, func, reg, bytes)); 163 } 164 165 /* 166 * Read configuration space register 167 */ 168 u_int32_t 169 pci_cfgregread(int bus, int slot, int func, int reg, int bytes) 170 { 171 uint32_t line; 172 173 /* 174 * Some BIOS writers seem to want to ignore the spec and put 175 * 0 in the intline rather than 255 to indicate none. The rest of 176 * the code uses 255 as an invalid IRQ. 177 */ 178 if (reg == PCIR_INTLINE && bytes == 1) { 179 line = pci_docfgregread(bus, slot, func, PCIR_INTLINE, 1); 180 return (pci_i386_map_intline(line)); 181 } 182 return (pci_docfgregread(bus, slot, func, reg, bytes)); 183 } 184 185 /* 186 * Write configuration space register 187 */ 188 void 189 pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes) 190 { 191 192 if (cfgmech == CFGMECH_PCIE && 193 (bus >= pcie_minbus && bus <= pcie_maxbus) && 194 (bus != 0 || !(1 << slot & pcie_badslots))) 195 pciereg_cfgwrite(bus, slot, func, reg, data, bytes); 196 else 197 pcireg_cfgwrite(bus, slot, func, reg, data, bytes); 198 } 199 200 /* 201 * Configuration space access using direct register operations 202 */ 203 204 /* enable configuration space accesses and return data port address */ 205 static int 206 pci_cfgenable(unsigned bus, unsigned slot, unsigned func, int reg, int bytes) 207 { 208 int dataport = 0; 209 210 if (bus <= PCI_BUSMAX 211 && slot < devmax 212 && func <= PCI_FUNCMAX 213 && (unsigned)reg <= PCI_REGMAX 214 && bytes != 3 215 && (unsigned)bytes <= 4 216 && (reg & (bytes - 1)) == 0) { 217 switch (cfgmech) { 218 case CFGMECH_PCIE: 219 case CFGMECH_1: 220 outl(CONF1_ADDR_PORT, (1U << 31) 221 | (bus << 16) | (slot << 11) 222 | (func << 8) | (reg & ~0x03)); 223 dataport = CONF1_DATA_PORT + (reg & 0x03); 224 break; 225 case CFGMECH_2: 226 outb(CONF2_ENABLE_PORT, 0xf0 | (func << 1)); 227 outb(CONF2_FORWARD_PORT, bus); 228 dataport = 0xc000 | (slot << 8) | reg; 229 break; 230 } 231 } 232 return (dataport); 233 } 234 235 /* disable configuration space accesses */ 236 static void 237 pci_cfgdisable(void) 238 { 239 switch (cfgmech) { 240 case CFGMECH_PCIE: 241 case CFGMECH_1: 242 /* 243 * Do nothing for the config mechanism 1 case. 244 * Writing a 0 to the address port can apparently 245 * confuse some bridges and cause spurious 246 * access failures. 247 */ 248 break; 249 case CFGMECH_2: 250 outb(CONF2_ENABLE_PORT, 0); 251 break; 252 } 253 } 254 255 static int 256 pcireg_cfgread(int bus, int slot, int func, int reg, int bytes) 257 { 258 int data = -1; 259 int port; 260 261 mtx_lock_spin(&pcicfg_mtx); 262 port = pci_cfgenable(bus, slot, func, reg, bytes); 263 if (port != 0) { 264 switch (bytes) { 265 case 1: 266 data = inb(port); 267 break; 268 case 2: 269 data = inw(port); 270 break; 271 case 4: 272 data = inl(port); 273 break; 274 } 275 pci_cfgdisable(); 276 } 277 mtx_unlock_spin(&pcicfg_mtx); 278 return (data); 279 } 280 281 static void 282 pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes) 283 { 284 int port; 285 286 mtx_lock_spin(&pcicfg_mtx); 287 port = pci_cfgenable(bus, slot, func, reg, bytes); 288 if (port != 0) { 289 switch (bytes) { 290 case 1: 291 outb(port, data); 292 break; 293 case 2: 294 outw(port, data); 295 break; 296 case 4: 297 outl(port, data); 298 break; 299 } 300 pci_cfgdisable(); 301 } 302 mtx_unlock_spin(&pcicfg_mtx); 303 } 304 305 /* check whether the configuration mechanism has been correctly identified */ 306 static int 307 pci_cfgcheck(int maxdev) 308 { 309 uint32_t id, class; 310 uint8_t header; 311 uint8_t device; 312 int port; 313 314 if (bootverbose) 315 printf("pci_cfgcheck:\tdevice "); 316 317 for (device = 0; device < maxdev; device++) { 318 if (bootverbose) 319 printf("%d ", device); 320 321 port = pci_cfgenable(0, device, 0, 0, 4); 322 id = inl(port); 323 if (id == 0 || id == 0xffffffff) 324 continue; 325 326 port = pci_cfgenable(0, device, 0, 8, 4); 327 class = inl(port) >> 8; 328 if (bootverbose) 329 printf("[class=%06x] ", class); 330 if (class == 0 || (class & 0xf870ff) != 0) 331 continue; 332 333 port = pci_cfgenable(0, device, 0, 14, 1); 334 header = inb(port); 335 if (bootverbose) 336 printf("[hdr=%02x] ", header); 337 if ((header & 0x7e) != 0) 338 continue; 339 340 if (bootverbose) 341 printf("is there (id=%08x)\n", id); 342 343 pci_cfgdisable(); 344 return (1); 345 } 346 if (bootverbose) 347 printf("-- nothing found\n"); 348 349 pci_cfgdisable(); 350 return (0); 351 } 352 353 static int 354 pcireg_cfgopen(void) 355 { 356 uint32_t mode1res, oldval1; 357 uint8_t mode2res, oldval2; 358 359 /* Check for type #1 first. */ 360 oldval1 = inl(CONF1_ADDR_PORT); 361 362 if (bootverbose) { 363 printf("pci_open(1):\tmode 1 addr port (0x0cf8) is 0x%08x\n", 364 oldval1); 365 } 366 367 cfgmech = CFGMECH_1; 368 devmax = 32; 369 370 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK); 371 DELAY(1); 372 mode1res = inl(CONF1_ADDR_PORT); 373 outl(CONF1_ADDR_PORT, oldval1); 374 375 if (bootverbose) 376 printf("pci_open(1a):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 377 CONF1_ENABLE_CHK); 378 379 if (mode1res) { 380 if (pci_cfgcheck(32)) 381 return (cfgmech); 382 } 383 384 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK1); 385 mode1res = inl(CONF1_ADDR_PORT); 386 outl(CONF1_ADDR_PORT, oldval1); 387 388 if (bootverbose) 389 printf("pci_open(1b):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 390 CONF1_ENABLE_CHK1); 391 392 if ((mode1res & CONF1_ENABLE_MSK1) == CONF1_ENABLE_RES1) { 393 if (pci_cfgcheck(32)) 394 return (cfgmech); 395 } 396 397 /* Type #1 didn't work, so try type #2. */ 398 oldval2 = inb(CONF2_ENABLE_PORT); 399 400 if (bootverbose) { 401 printf("pci_open(2):\tmode 2 enable port (0x0cf8) is 0x%02x\n", 402 oldval2); 403 } 404 405 if ((oldval2 & 0xf0) == 0) { 406 cfgmech = CFGMECH_2; 407 devmax = 16; 408 409 outb(CONF2_ENABLE_PORT, CONF2_ENABLE_CHK); 410 mode2res = inb(CONF2_ENABLE_PORT); 411 outb(CONF2_ENABLE_PORT, oldval2); 412 413 if (bootverbose) 414 printf("pci_open(2a):\tmode2res=0x%02x (0x%02x)\n", 415 mode2res, CONF2_ENABLE_CHK); 416 417 if (mode2res == CONF2_ENABLE_RES) { 418 if (bootverbose) 419 printf("pci_open(2a):\tnow trying mechanism 2\n"); 420 421 if (pci_cfgcheck(16)) 422 return (cfgmech); 423 } 424 } 425 426 /* Nothing worked, so punt. */ 427 cfgmech = CFGMECH_NONE; 428 devmax = 0; 429 return (cfgmech); 430 } 431 432 int 433 pcie_cfgregopen(uint64_t base, uint8_t minbus, uint8_t maxbus) 434 { 435 struct pcie_cfg_list *pcielist; 436 struct pcie_cfg_elem *pcie_array, *elem; 437 #ifdef SMP 438 struct pcpu *pc; 439 #endif 440 vm_offset_t va; 441 uint32_t val1, val2; 442 int i, slot; 443 444 if (!mcfg_enable) 445 return (0); 446 447 if (minbus != 0) 448 return (0); 449 450 if (!pae_mode && base >= 0x100000000) { 451 if (bootverbose) 452 printf( 453 "PCI: Memory Mapped PCI configuration area base 0x%jx too high\n", 454 (uintmax_t)base); 455 return (0); 456 } 457 458 if (bootverbose) 459 printf("PCIe: Memory Mapped configuration base @ 0x%jx\n", 460 (uintmax_t)base); 461 462 #ifdef SMP 463 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) 464 #endif 465 { 466 pcie_array = malloc(sizeof(struct pcie_cfg_elem) * PCIE_CACHE, 467 M_DEVBUF, M_NOWAIT); 468 if (pcie_array == NULL) 469 return (0); 470 471 va = kva_alloc(PCIE_CACHE * PAGE_SIZE); 472 if (va == 0) { 473 free(pcie_array, M_DEVBUF); 474 return (0); 475 } 476 477 #ifdef SMP 478 pcielist = &pcie_list[pc->pc_cpuid]; 479 #else 480 pcielist = &pcie_list[0]; 481 #endif 482 TAILQ_INIT(pcielist); 483 for (i = 0; i < PCIE_CACHE; i++) { 484 elem = &pcie_array[i]; 485 elem->vapage = va + (i * PAGE_SIZE); 486 elem->papage = 0; 487 TAILQ_INSERT_HEAD(pcielist, elem, elem); 488 } 489 } 490 491 pcie_base = base; 492 pcie_minbus = minbus; 493 pcie_maxbus = maxbus; 494 cfgmech = CFGMECH_PCIE; 495 devmax = 32; 496 497 /* 498 * On some AMD systems, some of the devices on bus 0 are 499 * inaccessible using memory-mapped PCI config access. Walk 500 * bus 0 looking for such devices. For these devices, we will 501 * fall back to using type 1 config access instead. 502 */ 503 if (pci_cfgregopen() != 0) { 504 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 505 val1 = pcireg_cfgread(0, slot, 0, 0, 4); 506 if (val1 == 0xffffffff) 507 continue; 508 509 val2 = pciereg_cfgread(0, slot, 0, 0, 4); 510 if (val2 != val1) 511 pcie_badslots |= (1 << slot); 512 } 513 } 514 515 return (1); 516 } 517 518 #define PCIE_PADDR(base, reg, bus, slot, func) \ 519 ((base) + \ 520 ((((bus) & 0xff) << 20) | \ 521 (((slot) & 0x1f) << 15) | \ 522 (((func) & 0x7) << 12) | \ 523 ((reg) & 0xfff))) 524 525 static __inline vm_offset_t 526 pciereg_findaddr(int bus, unsigned slot, unsigned func, unsigned reg) 527 { 528 struct pcie_cfg_list *pcielist; 529 struct pcie_cfg_elem *elem; 530 vm_paddr_t pa, papage; 531 532 pa = PCIE_PADDR(pcie_base, reg, bus, slot, func); 533 papage = pa & ~PAGE_MASK; 534 535 /* 536 * Find an element in the cache that matches the physical page desired, 537 * or create a new mapping from the least recently used element. 538 * A very simple LRU algorithm is used here, does it need to be more 539 * efficient? 540 */ 541 pcielist = &pcie_list[PCPU_GET(cpuid)]; 542 TAILQ_FOREACH(elem, pcielist, elem) { 543 if (elem->papage == papage) 544 break; 545 } 546 547 if (elem == NULL) { 548 elem = TAILQ_LAST(pcielist, pcie_cfg_list); 549 if (elem->papage != 0) { 550 pmap_kremove(elem->vapage); 551 invlpg(elem->vapage); 552 } 553 pmap_kenter(elem->vapage, papage); 554 elem->papage = papage; 555 } 556 557 if (elem != TAILQ_FIRST(pcielist)) { 558 TAILQ_REMOVE(pcielist, elem, elem); 559 TAILQ_INSERT_HEAD(pcielist, elem, elem); 560 } 561 return (elem->vapage | (pa & PAGE_MASK)); 562 } 563 564 /* 565 * AMD BIOS And Kernel Developer's Guides for CPU families starting with 10h 566 * have a requirement that all accesses to the memory mapped PCI configuration 567 * space are done using AX class of registers. 568 * Since other vendors do not currently have any contradicting requirements 569 * the AMD access pattern is applied universally. 570 */ 571 572 static int 573 pciereg_cfgread(int bus, unsigned slot, unsigned func, unsigned reg, 574 unsigned bytes) 575 { 576 vm_offset_t va; 577 int data = -1; 578 579 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 580 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 581 return (-1); 582 583 critical_enter(); 584 va = pciereg_findaddr(bus, slot, func, reg); 585 586 switch (bytes) { 587 case 4: 588 __asm("movl %1, %0" : "=a" (data) 589 : "m" (*(volatile uint32_t *)va)); 590 break; 591 case 2: 592 __asm("movzwl %1, %0" : "=a" (data) 593 : "m" (*(volatile uint16_t *)va)); 594 break; 595 case 1: 596 __asm("movzbl %1, %0" : "=a" (data) 597 : "m" (*(volatile uint8_t *)va)); 598 break; 599 } 600 601 critical_exit(); 602 return (data); 603 } 604 605 static void 606 pciereg_cfgwrite(int bus, unsigned slot, unsigned func, unsigned reg, int data, 607 unsigned bytes) 608 { 609 vm_offset_t va; 610 611 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 612 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 613 return; 614 615 critical_enter(); 616 va = pciereg_findaddr(bus, slot, func, reg); 617 618 switch (bytes) { 619 case 4: 620 __asm("movl %1, %0" : "=m" (*(volatile uint32_t *)va) 621 : "a" (data)); 622 break; 623 case 2: 624 __asm("movw %1, %0" : "=m" (*(volatile uint16_t *)va) 625 : "a" ((uint16_t)data)); 626 break; 627 case 1: 628 __asm("movb %1, %0" : "=m" (*(volatile uint8_t *)va) 629 : "a" ((uint8_t)data)); 630 break; 631 } 632 633 critical_exit(); 634 } 635