1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org> 6 * Copyright (c) 2000, BSDi 7 * Copyright (c) 2004, Scott Long <scottl@freebsd.org> 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/bus.h> 35 #include <sys/lock.h> 36 #include <sys/kernel.h> 37 #include <sys/mutex.h> 38 #include <sys/malloc.h> 39 #include <sys/queue.h> 40 #include <sys/sysctl.h> 41 #include <dev/pci/pcivar.h> 42 #include <dev/pci/pcireg.h> 43 #include <machine/pci_cfgreg.h> 44 #include <machine/pc/bios.h> 45 46 #include <vm/vm.h> 47 #include <vm/vm_param.h> 48 #include <vm/vm_kern.h> 49 #include <vm/vm_extern.h> 50 #include <vm/pmap.h> 51 52 #define PRVERB(a) do { \ 53 if (bootverbose) \ 54 printf a ; \ 55 } while(0) 56 57 #define PCIE_CACHE 8 58 struct pcie_cfg_elem { 59 TAILQ_ENTRY(pcie_cfg_elem) elem; 60 vm_offset_t vapage; 61 vm_paddr_t papage; 62 }; 63 64 SYSCTL_DECL(_hw_pci); 65 66 static TAILQ_HEAD(pcie_cfg_list, pcie_cfg_elem) pcie_list[MAXCPU]; 67 static uint64_t pcie_base; 68 static int pcie_minbus, pcie_maxbus; 69 static uint32_t pcie_badslots; 70 int cfgmech; 71 static int devmax; 72 static struct mtx pcicfg_mtx; 73 static int mcfg_enable = 1; 74 SYSCTL_INT(_hw_pci, OID_AUTO, mcfg, CTLFLAG_RDTUN, &mcfg_enable, 0, 75 "Enable support for PCI-e memory mapped config access"); 76 77 static uint32_t pci_docfgregread(int bus, int slot, int func, int reg, 78 int bytes); 79 static int pcireg_cfgread(int bus, int slot, int func, int reg, int bytes); 80 static void pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes); 81 static int pcireg_cfgopen(void); 82 static int pciereg_cfgread(int bus, unsigned slot, unsigned func, 83 unsigned reg, unsigned bytes); 84 static void pciereg_cfgwrite(int bus, unsigned slot, unsigned func, 85 unsigned reg, int data, unsigned bytes); 86 87 /* 88 * Some BIOS writers seem to want to ignore the spec and put 89 * 0 in the intline rather than 255 to indicate none. Some use 90 * numbers in the range 128-254 to indicate something strange and 91 * apparently undocumented anywhere. Assume these are completely bogus 92 * and map them to 255, which means "none". 93 */ 94 static __inline int 95 pci_i386_map_intline(int line) 96 { 97 if (line == 0 || line >= 128) 98 return (PCI_INVALID_IRQ); 99 return (line); 100 } 101 102 static u_int16_t 103 pcibios_get_version(void) 104 { 105 struct bios_regs args; 106 107 if (PCIbios.ventry == 0) { 108 PRVERB(("pcibios: No call entry point\n")); 109 return (0); 110 } 111 args.eax = PCIBIOS_BIOS_PRESENT; 112 if (bios32(&args, PCIbios.ventry, GSEL(GCODE_SEL, SEL_KPL))) { 113 PRVERB(("pcibios: BIOS_PRESENT call failed\n")); 114 return (0); 115 } 116 if (args.edx != 0x20494350) { 117 PRVERB(("pcibios: BIOS_PRESENT didn't return 'PCI ' in edx\n")); 118 return (0); 119 } 120 return (args.ebx & 0xffff); 121 } 122 123 /* 124 * Initialise access to PCI configuration space 125 */ 126 int 127 pci_cfgregopen(void) 128 { 129 uint16_t v; 130 static int opened = 0; 131 132 if (opened) 133 return (1); 134 135 if (cfgmech == CFGMECH_NONE && pcireg_cfgopen() == 0) 136 return (0); 137 138 v = pcibios_get_version(); 139 if (v > 0) 140 PRVERB(("pcibios: BIOS version %x.%02x\n", (v & 0xff00) >> 8, 141 v & 0xff)); 142 mtx_init(&pcicfg_mtx, "pcicfg", NULL, MTX_SPIN); 143 opened = 1; 144 145 /* $PIR requires PCI BIOS 2.10 or greater. */ 146 if (v >= 0x0210) 147 pci_pir_open(); 148 149 return (1); 150 } 151 152 static uint32_t 153 pci_docfgregread(int bus, int slot, int func, int reg, int bytes) 154 { 155 156 if (cfgmech == CFGMECH_PCIE && 157 (bus >= pcie_minbus && bus <= pcie_maxbus) && 158 (bus != 0 || !(1 << slot & pcie_badslots))) 159 return (pciereg_cfgread(bus, slot, func, reg, bytes)); 160 else 161 return (pcireg_cfgread(bus, slot, func, reg, bytes)); 162 } 163 164 /* 165 * Read configuration space register 166 */ 167 u_int32_t 168 pci_cfgregread(int bus, int slot, int func, int reg, int bytes) 169 { 170 uint32_t line; 171 172 /* 173 * Some BIOS writers seem to want to ignore the spec and put 174 * 0 in the intline rather than 255 to indicate none. The rest of 175 * the code uses 255 as an invalid IRQ. 176 */ 177 if (reg == PCIR_INTLINE && bytes == 1) { 178 line = pci_docfgregread(bus, slot, func, PCIR_INTLINE, 1); 179 return (pci_i386_map_intline(line)); 180 } 181 return (pci_docfgregread(bus, slot, func, reg, bytes)); 182 } 183 184 /* 185 * Write configuration space register 186 */ 187 void 188 pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes) 189 { 190 191 if (cfgmech == CFGMECH_PCIE && 192 (bus >= pcie_minbus && bus <= pcie_maxbus) && 193 (bus != 0 || !(1 << slot & pcie_badslots))) 194 pciereg_cfgwrite(bus, slot, func, reg, data, bytes); 195 else 196 pcireg_cfgwrite(bus, slot, func, reg, data, bytes); 197 } 198 199 /* 200 * Configuration space access using direct register operations 201 */ 202 203 /* enable configuration space accesses and return data port address */ 204 static int 205 pci_cfgenable(unsigned bus, unsigned slot, unsigned func, int reg, int bytes) 206 { 207 int dataport = 0; 208 209 if (bus <= PCI_BUSMAX 210 && slot < devmax 211 && func <= PCI_FUNCMAX 212 && (unsigned)reg <= PCI_REGMAX 213 && bytes != 3 214 && (unsigned)bytes <= 4 215 && (reg & (bytes - 1)) == 0) { 216 switch (cfgmech) { 217 case CFGMECH_PCIE: 218 case CFGMECH_1: 219 outl(CONF1_ADDR_PORT, (1U << 31) 220 | (bus << 16) | (slot << 11) 221 | (func << 8) | (reg & ~0x03)); 222 dataport = CONF1_DATA_PORT + (reg & 0x03); 223 break; 224 case CFGMECH_2: 225 outb(CONF2_ENABLE_PORT, 0xf0 | (func << 1)); 226 outb(CONF2_FORWARD_PORT, bus); 227 dataport = 0xc000 | (slot << 8) | reg; 228 break; 229 } 230 } 231 return (dataport); 232 } 233 234 /* disable configuration space accesses */ 235 static void 236 pci_cfgdisable(void) 237 { 238 switch (cfgmech) { 239 case CFGMECH_PCIE: 240 case CFGMECH_1: 241 /* 242 * Do nothing for the config mechanism 1 case. 243 * Writing a 0 to the address port can apparently 244 * confuse some bridges and cause spurious 245 * access failures. 246 */ 247 break; 248 case CFGMECH_2: 249 outb(CONF2_ENABLE_PORT, 0); 250 break; 251 } 252 } 253 254 static int 255 pcireg_cfgread(int bus, int slot, int func, int reg, int bytes) 256 { 257 int data = -1; 258 int port; 259 260 mtx_lock_spin(&pcicfg_mtx); 261 port = pci_cfgenable(bus, slot, func, reg, bytes); 262 if (port != 0) { 263 switch (bytes) { 264 case 1: 265 data = inb(port); 266 break; 267 case 2: 268 data = inw(port); 269 break; 270 case 4: 271 data = inl(port); 272 break; 273 } 274 pci_cfgdisable(); 275 } 276 mtx_unlock_spin(&pcicfg_mtx); 277 return (data); 278 } 279 280 static void 281 pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes) 282 { 283 int port; 284 285 mtx_lock_spin(&pcicfg_mtx); 286 port = pci_cfgenable(bus, slot, func, reg, bytes); 287 if (port != 0) { 288 switch (bytes) { 289 case 1: 290 outb(port, data); 291 break; 292 case 2: 293 outw(port, data); 294 break; 295 case 4: 296 outl(port, data); 297 break; 298 } 299 pci_cfgdisable(); 300 } 301 mtx_unlock_spin(&pcicfg_mtx); 302 } 303 304 /* check whether the configuration mechanism has been correctly identified */ 305 static int 306 pci_cfgcheck(int maxdev) 307 { 308 uint32_t id, class; 309 uint8_t header; 310 uint8_t device; 311 int port; 312 313 if (bootverbose) 314 printf("pci_cfgcheck:\tdevice "); 315 316 for (device = 0; device < maxdev; device++) { 317 if (bootverbose) 318 printf("%d ", device); 319 320 port = pci_cfgenable(0, device, 0, 0, 4); 321 id = inl(port); 322 if (id == 0 || id == 0xffffffff) 323 continue; 324 325 port = pci_cfgenable(0, device, 0, 8, 4); 326 class = inl(port) >> 8; 327 if (bootverbose) 328 printf("[class=%06x] ", class); 329 if (class == 0 || (class & 0xf870ff) != 0) 330 continue; 331 332 port = pci_cfgenable(0, device, 0, 14, 1); 333 header = inb(port); 334 if (bootverbose) 335 printf("[hdr=%02x] ", header); 336 if ((header & 0x7e) != 0) 337 continue; 338 339 if (bootverbose) 340 printf("is there (id=%08x)\n", id); 341 342 pci_cfgdisable(); 343 return (1); 344 } 345 if (bootverbose) 346 printf("-- nothing found\n"); 347 348 pci_cfgdisable(); 349 return (0); 350 } 351 352 static int 353 pcireg_cfgopen(void) 354 { 355 uint32_t mode1res, oldval1; 356 uint8_t mode2res, oldval2; 357 358 /* Check for type #1 first. */ 359 oldval1 = inl(CONF1_ADDR_PORT); 360 361 if (bootverbose) { 362 printf("pci_open(1):\tmode 1 addr port (0x0cf8) is 0x%08x\n", 363 oldval1); 364 } 365 366 cfgmech = CFGMECH_1; 367 devmax = 32; 368 369 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK); 370 DELAY(1); 371 mode1res = inl(CONF1_ADDR_PORT); 372 outl(CONF1_ADDR_PORT, oldval1); 373 374 if (bootverbose) 375 printf("pci_open(1a):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 376 CONF1_ENABLE_CHK); 377 378 if (mode1res) { 379 if (pci_cfgcheck(32)) 380 return (cfgmech); 381 } 382 383 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK1); 384 mode1res = inl(CONF1_ADDR_PORT); 385 outl(CONF1_ADDR_PORT, oldval1); 386 387 if (bootverbose) 388 printf("pci_open(1b):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 389 CONF1_ENABLE_CHK1); 390 391 if ((mode1res & CONF1_ENABLE_MSK1) == CONF1_ENABLE_RES1) { 392 if (pci_cfgcheck(32)) 393 return (cfgmech); 394 } 395 396 /* Type #1 didn't work, so try type #2. */ 397 oldval2 = inb(CONF2_ENABLE_PORT); 398 399 if (bootverbose) { 400 printf("pci_open(2):\tmode 2 enable port (0x0cf8) is 0x%02x\n", 401 oldval2); 402 } 403 404 if ((oldval2 & 0xf0) == 0) { 405 cfgmech = CFGMECH_2; 406 devmax = 16; 407 408 outb(CONF2_ENABLE_PORT, CONF2_ENABLE_CHK); 409 mode2res = inb(CONF2_ENABLE_PORT); 410 outb(CONF2_ENABLE_PORT, oldval2); 411 412 if (bootverbose) 413 printf("pci_open(2a):\tmode2res=0x%02x (0x%02x)\n", 414 mode2res, CONF2_ENABLE_CHK); 415 416 if (mode2res == CONF2_ENABLE_RES) { 417 if (bootverbose) 418 printf("pci_open(2a):\tnow trying mechanism 2\n"); 419 420 if (pci_cfgcheck(16)) 421 return (cfgmech); 422 } 423 } 424 425 /* Nothing worked, so punt. */ 426 cfgmech = CFGMECH_NONE; 427 devmax = 0; 428 return (cfgmech); 429 } 430 431 int 432 pcie_cfgregopen(uint64_t base, uint8_t minbus, uint8_t maxbus) 433 { 434 struct pcie_cfg_list *pcielist; 435 struct pcie_cfg_elem *pcie_array, *elem; 436 #ifdef SMP 437 struct pcpu *pc; 438 #endif 439 vm_offset_t va; 440 uint32_t val1, val2; 441 int i, slot; 442 443 if (!mcfg_enable) 444 return (0); 445 446 if (minbus != 0) 447 return (0); 448 449 if (!pae_mode && base >= 0x100000000) { 450 if (bootverbose) 451 printf( 452 "PCI: Memory Mapped PCI configuration area base 0x%jx too high\n", 453 (uintmax_t)base); 454 return (0); 455 } 456 457 if (bootverbose) 458 printf("PCIe: Memory Mapped configuration base @ 0x%jx\n", 459 (uintmax_t)base); 460 461 #ifdef SMP 462 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) 463 #endif 464 { 465 pcie_array = malloc(sizeof(struct pcie_cfg_elem) * PCIE_CACHE, 466 M_DEVBUF, M_NOWAIT); 467 if (pcie_array == NULL) 468 return (0); 469 470 va = kva_alloc(PCIE_CACHE * PAGE_SIZE); 471 if (va == 0) { 472 free(pcie_array, M_DEVBUF); 473 return (0); 474 } 475 476 #ifdef SMP 477 pcielist = &pcie_list[pc->pc_cpuid]; 478 #else 479 pcielist = &pcie_list[0]; 480 #endif 481 TAILQ_INIT(pcielist); 482 for (i = 0; i < PCIE_CACHE; i++) { 483 elem = &pcie_array[i]; 484 elem->vapage = va + (i * PAGE_SIZE); 485 elem->papage = 0; 486 TAILQ_INSERT_HEAD(pcielist, elem, elem); 487 } 488 } 489 490 pcie_base = base; 491 pcie_minbus = minbus; 492 pcie_maxbus = maxbus; 493 cfgmech = CFGMECH_PCIE; 494 devmax = 32; 495 496 /* 497 * On some AMD systems, some of the devices on bus 0 are 498 * inaccessible using memory-mapped PCI config access. Walk 499 * bus 0 looking for such devices. For these devices, we will 500 * fall back to using type 1 config access instead. 501 */ 502 if (pci_cfgregopen() != 0) { 503 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 504 val1 = pcireg_cfgread(0, slot, 0, 0, 4); 505 if (val1 == 0xffffffff) 506 continue; 507 508 val2 = pciereg_cfgread(0, slot, 0, 0, 4); 509 if (val2 != val1) 510 pcie_badslots |= (1 << slot); 511 } 512 } 513 514 return (1); 515 } 516 517 #define PCIE_PADDR(base, reg, bus, slot, func) \ 518 ((base) + \ 519 ((((bus) & 0xff) << 20) | \ 520 (((slot) & 0x1f) << 15) | \ 521 (((func) & 0x7) << 12) | \ 522 ((reg) & 0xfff))) 523 524 static __inline vm_offset_t 525 pciereg_findaddr(int bus, unsigned slot, unsigned func, unsigned reg) 526 { 527 struct pcie_cfg_list *pcielist; 528 struct pcie_cfg_elem *elem; 529 vm_paddr_t pa, papage; 530 531 pa = PCIE_PADDR(pcie_base, reg, bus, slot, func); 532 papage = pa & ~PAGE_MASK; 533 534 /* 535 * Find an element in the cache that matches the physical page desired, 536 * or create a new mapping from the least recently used element. 537 * A very simple LRU algorithm is used here, does it need to be more 538 * efficient? 539 */ 540 pcielist = &pcie_list[PCPU_GET(cpuid)]; 541 TAILQ_FOREACH(elem, pcielist, elem) { 542 if (elem->papage == papage) 543 break; 544 } 545 546 if (elem == NULL) { 547 elem = TAILQ_LAST(pcielist, pcie_cfg_list); 548 if (elem->papage != 0) { 549 pmap_kremove(elem->vapage); 550 invlpg(elem->vapage); 551 } 552 pmap_kenter(elem->vapage, papage); 553 elem->papage = papage; 554 } 555 556 if (elem != TAILQ_FIRST(pcielist)) { 557 TAILQ_REMOVE(pcielist, elem, elem); 558 TAILQ_INSERT_HEAD(pcielist, elem, elem); 559 } 560 return (elem->vapage | (pa & PAGE_MASK)); 561 } 562 563 /* 564 * AMD BIOS And Kernel Developer's Guides for CPU families starting with 10h 565 * have a requirement that all accesses to the memory mapped PCI configuration 566 * space are done using AX class of registers. 567 * Since other vendors do not currently have any contradicting requirements 568 * the AMD access pattern is applied universally. 569 */ 570 571 static int 572 pciereg_cfgread(int bus, unsigned slot, unsigned func, unsigned reg, 573 unsigned bytes) 574 { 575 vm_offset_t va; 576 int data = -1; 577 578 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 579 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 580 return (-1); 581 582 critical_enter(); 583 va = pciereg_findaddr(bus, slot, func, reg); 584 585 switch (bytes) { 586 case 4: 587 __asm("movl %1, %0" : "=a" (data) 588 : "m" (*(volatile uint32_t *)va)); 589 break; 590 case 2: 591 __asm("movzwl %1, %0" : "=a" (data) 592 : "m" (*(volatile uint16_t *)va)); 593 break; 594 case 1: 595 __asm("movzbl %1, %0" : "=a" (data) 596 : "m" (*(volatile uint8_t *)va)); 597 break; 598 } 599 600 critical_exit(); 601 return (data); 602 } 603 604 static void 605 pciereg_cfgwrite(int bus, unsigned slot, unsigned func, unsigned reg, int data, 606 unsigned bytes) 607 { 608 vm_offset_t va; 609 610 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 611 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 612 return; 613 614 critical_enter(); 615 va = pciereg_findaddr(bus, slot, func, reg); 616 617 switch (bytes) { 618 case 4: 619 __asm("movl %1, %0" : "=m" (*(volatile uint32_t *)va) 620 : "a" (data)); 621 break; 622 case 2: 623 __asm("movw %1, %0" : "=m" (*(volatile uint16_t *)va) 624 : "a" ((uint16_t)data)); 625 break; 626 case 1: 627 __asm("movb %1, %0" : "=m" (*(volatile uint8_t *)va) 628 : "a" ((uint8_t)data)); 629 break; 630 } 631 632 critical_exit(); 633 } 634