1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/pci.h> 3 #include <linux/module.h> 4 #include <linux/slab.h> 5 #include <linux/ioport.h> 6 #include <linux/wait.h> 7 8 #include "pci.h" 9 10 /* 11 * This interrupt-safe spinlock protects all accesses to PCI 12 * configuration space. 13 */ 14 15 DEFINE_RAW_SPINLOCK(pci_lock); 16 17 /* 18 * Wrappers for all PCI configuration access functions. They just check 19 * alignment, do locking and call the low-level functions pointed to 20 * by pci_dev->ops. 21 */ 22 23 #define PCI_byte_BAD 0 24 #define PCI_word_BAD (pos & 1) 25 #define PCI_dword_BAD (pos & 3) 26 27 #ifdef CONFIG_PCI_LOCKLESS_CONFIG 28 # define pci_lock_config(f) do { (void)(f); } while (0) 29 # define pci_unlock_config(f) do { (void)(f); } while (0) 30 #else 31 # define pci_lock_config(f) raw_spin_lock_irqsave(&pci_lock, f) 32 # define pci_unlock_config(f) raw_spin_unlock_irqrestore(&pci_lock, f) 33 #endif 34 35 #define PCI_OP_READ(size, type, len) \ 36 int noinline pci_bus_read_config_##size \ 37 (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \ 38 { \ 39 unsigned long flags; \ 40 u32 data = 0; \ 41 int res; \ 42 \ 43 if (PCI_##size##_BAD) \ 44 return PCIBIOS_BAD_REGISTER_NUMBER; \ 45 \ 46 pci_lock_config(flags); \ 47 res = bus->ops->read(bus, devfn, pos, len, &data); \ 48 if (res) \ 49 PCI_SET_ERROR_RESPONSE(value); \ 50 else \ 51 *value = (type)data; \ 52 pci_unlock_config(flags); \ 53 \ 54 return res; \ 55 } 56 57 #define PCI_OP_WRITE(size, type, len) \ 58 int noinline pci_bus_write_config_##size \ 59 (struct pci_bus *bus, unsigned int devfn, int pos, type value) \ 60 { \ 61 unsigned long flags; \ 62 int res; \ 63 \ 64 if (PCI_##size##_BAD) \ 65 return PCIBIOS_BAD_REGISTER_NUMBER; \ 66 \ 67 pci_lock_config(flags); \ 68 res = bus->ops->write(bus, devfn, pos, len, value); \ 69 pci_unlock_config(flags); \ 70 \ 71 return res; \ 72 } 73 74 PCI_OP_READ(byte, u8, 1) 75 PCI_OP_READ(word, u16, 2) 76 PCI_OP_READ(dword, u32, 4) 77 PCI_OP_WRITE(byte, u8, 1) 78 PCI_OP_WRITE(word, u16, 2) 79 PCI_OP_WRITE(dword, u32, 4) 80 81 EXPORT_SYMBOL(pci_bus_read_config_byte); 82 EXPORT_SYMBOL(pci_bus_read_config_word); 83 EXPORT_SYMBOL(pci_bus_read_config_dword); 84 EXPORT_SYMBOL(pci_bus_write_config_byte); 85 EXPORT_SYMBOL(pci_bus_write_config_word); 86 EXPORT_SYMBOL(pci_bus_write_config_dword); 87 88 int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, 89 int where, int size, u32 *val) 90 { 91 void __iomem *addr; 92 93 addr = bus->ops->map_bus(bus, devfn, where); 94 if (!addr) 95 return PCIBIOS_DEVICE_NOT_FOUND; 96 97 if (size == 1) 98 *val = readb(addr); 99 else if (size == 2) 100 *val = readw(addr); 101 else 102 *val = readl(addr); 103 104 return PCIBIOS_SUCCESSFUL; 105 } 106 EXPORT_SYMBOL_GPL(pci_generic_config_read); 107 108 int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, 109 int where, int size, u32 val) 110 { 111 void __iomem *addr; 112 113 addr = bus->ops->map_bus(bus, devfn, where); 114 if (!addr) 115 return PCIBIOS_DEVICE_NOT_FOUND; 116 117 if (size == 1) 118 writeb(val, addr); 119 else if (size == 2) 120 writew(val, addr); 121 else 122 writel(val, addr); 123 124 return PCIBIOS_SUCCESSFUL; 125 } 126 EXPORT_SYMBOL_GPL(pci_generic_config_write); 127 128 int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn, 129 int where, int size, u32 *val) 130 { 131 void __iomem *addr; 132 133 addr = bus->ops->map_bus(bus, devfn, where & ~0x3); 134 if (!addr) 135 return PCIBIOS_DEVICE_NOT_FOUND; 136 137 *val = readl(addr); 138 139 if (size <= 2) 140 *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); 141 142 return PCIBIOS_SUCCESSFUL; 143 } 144 EXPORT_SYMBOL_GPL(pci_generic_config_read32); 145 146 int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn, 147 int where, int size, u32 val) 148 { 149 void __iomem *addr; 150 u32 mask, tmp; 151 152 addr = bus->ops->map_bus(bus, devfn, where & ~0x3); 153 if (!addr) 154 return PCIBIOS_DEVICE_NOT_FOUND; 155 156 if (size == 4) { 157 writel(val, addr); 158 return PCIBIOS_SUCCESSFUL; 159 } 160 161 /* 162 * In general, hardware that supports only 32-bit writes on PCI is 163 * not spec-compliant. For example, software may perform a 16-bit 164 * write. If the hardware only supports 32-bit accesses, we must 165 * do a 32-bit read, merge in the 16 bits we intend to write, 166 * followed by a 32-bit write. If the 16 bits we *don't* intend to 167 * write happen to have any RW1C (write-one-to-clear) bits set, we 168 * just inadvertently cleared something we shouldn't have. 169 */ 170 if (!bus->unsafe_warn) { 171 dev_warn(&bus->dev, "%d-byte config write to %04x:%02x:%02x.%d offset %#x may corrupt adjacent RW1C bits\n", 172 size, pci_domain_nr(bus), bus->number, 173 PCI_SLOT(devfn), PCI_FUNC(devfn), where); 174 bus->unsafe_warn = 1; 175 } 176 177 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 178 tmp = readl(addr) & mask; 179 tmp |= val << ((where & 0x3) * 8); 180 writel(tmp, addr); 181 182 return PCIBIOS_SUCCESSFUL; 183 } 184 EXPORT_SYMBOL_GPL(pci_generic_config_write32); 185 186 /** 187 * pci_bus_set_ops - Set raw operations of pci bus 188 * @bus: pci bus struct 189 * @ops: new raw operations 190 * 191 * Return previous raw operations 192 */ 193 struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops) 194 { 195 struct pci_ops *old_ops; 196 unsigned long flags; 197 198 raw_spin_lock_irqsave(&pci_lock, flags); 199 old_ops = bus->ops; 200 bus->ops = ops; 201 raw_spin_unlock_irqrestore(&pci_lock, flags); 202 return old_ops; 203 } 204 EXPORT_SYMBOL(pci_bus_set_ops); 205 206 /* 207 * The following routines are to prevent the user from accessing PCI config 208 * space when it's unsafe to do so. Some devices require this during BIST and 209 * we're required to prevent it during D-state transitions. 210 * 211 * We have a bit per device to indicate it's blocked and a global wait queue 212 * for callers to sleep on until devices are unblocked. 213 */ 214 static DECLARE_WAIT_QUEUE_HEAD(pci_cfg_wait); 215 216 static noinline void pci_wait_cfg(struct pci_dev *dev) 217 __must_hold(&pci_lock) 218 { 219 do { 220 raw_spin_unlock_irq(&pci_lock); 221 wait_event(pci_cfg_wait, !dev->block_cfg_access); 222 raw_spin_lock_irq(&pci_lock); 223 } while (dev->block_cfg_access); 224 } 225 226 /* Returns 0 on success, negative values indicate error. */ 227 #define PCI_USER_READ_CONFIG(size, type) \ 228 int pci_user_read_config_##size \ 229 (struct pci_dev *dev, int pos, type *val) \ 230 { \ 231 u32 data = -1; \ 232 int ret; \ 233 \ 234 if (PCI_##size##_BAD) \ 235 return -EINVAL; \ 236 \ 237 raw_spin_lock_irq(&pci_lock); \ 238 if (unlikely(dev->block_cfg_access)) \ 239 pci_wait_cfg(dev); \ 240 ret = dev->bus->ops->read(dev->bus, dev->devfn, \ 241 pos, sizeof(type), &data); \ 242 raw_spin_unlock_irq(&pci_lock); \ 243 if (ret) \ 244 PCI_SET_ERROR_RESPONSE(val); \ 245 else \ 246 *val = (type)data; \ 247 \ 248 return pcibios_err_to_errno(ret); \ 249 } \ 250 EXPORT_SYMBOL_GPL(pci_user_read_config_##size); 251 252 /* Returns 0 on success, negative values indicate error. */ 253 #define PCI_USER_WRITE_CONFIG(size, type) \ 254 int pci_user_write_config_##size \ 255 (struct pci_dev *dev, int pos, type val) \ 256 { \ 257 int ret; \ 258 \ 259 if (PCI_##size##_BAD) \ 260 return -EINVAL; \ 261 \ 262 raw_spin_lock_irq(&pci_lock); \ 263 if (unlikely(dev->block_cfg_access)) \ 264 pci_wait_cfg(dev); \ 265 ret = dev->bus->ops->write(dev->bus, dev->devfn, \ 266 pos, sizeof(type), val); \ 267 raw_spin_unlock_irq(&pci_lock); \ 268 \ 269 return pcibios_err_to_errno(ret); \ 270 } \ 271 EXPORT_SYMBOL_GPL(pci_user_write_config_##size); 272 273 PCI_USER_READ_CONFIG(byte, u8) 274 PCI_USER_READ_CONFIG(word, u16) 275 PCI_USER_READ_CONFIG(dword, u32) 276 PCI_USER_WRITE_CONFIG(byte, u8) 277 PCI_USER_WRITE_CONFIG(word, u16) 278 PCI_USER_WRITE_CONFIG(dword, u32) 279 280 /** 281 * pci_cfg_access_lock - Lock PCI config reads/writes 282 * @dev: pci device struct 283 * 284 * When access is locked, any userspace reads or writes to config 285 * space and concurrent lock requests will sleep until access is 286 * allowed via pci_cfg_access_unlock() again. 287 */ 288 void pci_cfg_access_lock(struct pci_dev *dev) 289 { 290 might_sleep(); 291 292 lock_map_acquire(&dev->cfg_access_lock); 293 294 raw_spin_lock_irq(&pci_lock); 295 if (dev->block_cfg_access) 296 pci_wait_cfg(dev); 297 dev->block_cfg_access = 1; 298 raw_spin_unlock_irq(&pci_lock); 299 } 300 EXPORT_SYMBOL_GPL(pci_cfg_access_lock); 301 302 /** 303 * pci_cfg_access_trylock - try to lock PCI config reads/writes 304 * @dev: pci device struct 305 * 306 * Same as pci_cfg_access_lock, but will return 0 if access is 307 * already locked, 1 otherwise. This function can be used from 308 * atomic contexts. 309 */ 310 bool pci_cfg_access_trylock(struct pci_dev *dev) 311 { 312 unsigned long flags; 313 bool locked = true; 314 315 raw_spin_lock_irqsave(&pci_lock, flags); 316 if (dev->block_cfg_access) 317 locked = false; 318 else 319 dev->block_cfg_access = 1; 320 raw_spin_unlock_irqrestore(&pci_lock, flags); 321 322 return locked; 323 } 324 EXPORT_SYMBOL_GPL(pci_cfg_access_trylock); 325 326 /** 327 * pci_cfg_access_unlock - Unlock PCI config reads/writes 328 * @dev: pci device struct 329 * 330 * This function allows PCI config accesses to resume. 331 */ 332 void pci_cfg_access_unlock(struct pci_dev *dev) 333 { 334 unsigned long flags; 335 336 raw_spin_lock_irqsave(&pci_lock, flags); 337 338 /* 339 * This indicates a problem in the caller, but we don't need 340 * to kill them, unlike a double-block above. 341 */ 342 WARN_ON(!dev->block_cfg_access); 343 344 dev->block_cfg_access = 0; 345 raw_spin_unlock_irqrestore(&pci_lock, flags); 346 347 wake_up_all(&pci_cfg_wait); 348 349 lock_map_release(&dev->cfg_access_lock); 350 } 351 EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); 352 353 static inline int pcie_cap_version(const struct pci_dev *dev) 354 { 355 return pcie_caps_reg(dev) & PCI_EXP_FLAGS_VERS; 356 } 357 358 bool pcie_cap_has_lnkctl(const struct pci_dev *dev) 359 { 360 int type = pci_pcie_type(dev); 361 362 return type == PCI_EXP_TYPE_ENDPOINT || 363 type == PCI_EXP_TYPE_LEG_END || 364 type == PCI_EXP_TYPE_ROOT_PORT || 365 type == PCI_EXP_TYPE_UPSTREAM || 366 type == PCI_EXP_TYPE_DOWNSTREAM || 367 type == PCI_EXP_TYPE_PCI_BRIDGE || 368 type == PCI_EXP_TYPE_PCIE_BRIDGE; 369 } 370 371 bool pcie_cap_has_lnkctl2(const struct pci_dev *dev) 372 { 373 return pcie_cap_has_lnkctl(dev) && pcie_cap_version(dev) > 1; 374 } 375 376 static inline bool pcie_cap_has_sltctl(const struct pci_dev *dev) 377 { 378 return pcie_downstream_port(dev) && 379 pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT; 380 } 381 382 bool pcie_cap_has_rtctl(const struct pci_dev *dev) 383 { 384 int type = pci_pcie_type(dev); 385 386 return type == PCI_EXP_TYPE_ROOT_PORT || 387 type == PCI_EXP_TYPE_RC_EC; 388 } 389 390 static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos) 391 { 392 if (!pci_is_pcie(dev)) 393 return false; 394 395 switch (pos) { 396 case PCI_EXP_FLAGS: 397 return true; 398 case PCI_EXP_DEVCAP: 399 case PCI_EXP_DEVCTL: 400 case PCI_EXP_DEVSTA: 401 return true; 402 case PCI_EXP_LNKCAP: 403 case PCI_EXP_LNKCTL: 404 case PCI_EXP_LNKSTA: 405 return pcie_cap_has_lnkctl(dev); 406 case PCI_EXP_SLTCAP: 407 case PCI_EXP_SLTCTL: 408 case PCI_EXP_SLTSTA: 409 return pcie_cap_has_sltctl(dev); 410 case PCI_EXP_RTCTL: 411 case PCI_EXP_RTCAP: 412 case PCI_EXP_RTSTA: 413 return pcie_cap_has_rtctl(dev); 414 case PCI_EXP_DEVCAP2: 415 case PCI_EXP_DEVCTL2: 416 return pcie_cap_version(dev) > 1; 417 case PCI_EXP_LNKCAP2: 418 case PCI_EXP_LNKCTL2: 419 case PCI_EXP_LNKSTA2: 420 return pcie_cap_has_lnkctl2(dev); 421 default: 422 return false; 423 } 424 } 425 426 /* 427 * Note that these accessor functions are only for the "PCI Express 428 * Capability" (see PCIe spec r3.0, sec 7.8). They do not apply to the 429 * other "PCI Express Extended Capabilities" (AER, VC, ACS, MFVC, etc.) 430 */ 431 int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val) 432 { 433 int ret; 434 435 *val = 0; 436 if (pos & 1) 437 return PCIBIOS_BAD_REGISTER_NUMBER; 438 439 if (pcie_capability_reg_implemented(dev, pos)) { 440 ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val); 441 /* 442 * Reset *val to 0 if pci_read_config_word() fails; it may 443 * have been written as 0xFFFF (PCI_ERROR_RESPONSE) if the 444 * config read failed on PCI. 445 */ 446 if (ret) 447 *val = 0; 448 return ret; 449 } 450 451 /* 452 * For Functions that do not implement the Slot Capabilities, 453 * Slot Status, and Slot Control registers, these spaces must 454 * be hardwired to 0b, with the exception of the Presence Detect 455 * State bit in the Slot Status register of Downstream Ports, 456 * which must be hardwired to 1b. (PCIe Base Spec 3.0, sec 7.8) 457 */ 458 if (pci_is_pcie(dev) && pcie_downstream_port(dev) && 459 pos == PCI_EXP_SLTSTA) 460 *val = PCI_EXP_SLTSTA_PDS; 461 462 return 0; 463 } 464 EXPORT_SYMBOL(pcie_capability_read_word); 465 466 int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val) 467 { 468 int ret; 469 470 *val = 0; 471 if (pos & 3) 472 return PCIBIOS_BAD_REGISTER_NUMBER; 473 474 if (pcie_capability_reg_implemented(dev, pos)) { 475 ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val); 476 /* 477 * Reset *val to 0 if pci_read_config_dword() fails; it may 478 * have been written as 0xFFFFFFFF (PCI_ERROR_RESPONSE) if 479 * the config read failed on PCI. 480 */ 481 if (ret) 482 *val = 0; 483 return ret; 484 } 485 486 if (pci_is_pcie(dev) && pcie_downstream_port(dev) && 487 pos == PCI_EXP_SLTSTA) 488 *val = PCI_EXP_SLTSTA_PDS; 489 490 return 0; 491 } 492 EXPORT_SYMBOL(pcie_capability_read_dword); 493 494 int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val) 495 { 496 if (pos & 1) 497 return PCIBIOS_BAD_REGISTER_NUMBER; 498 499 if (!pcie_capability_reg_implemented(dev, pos)) 500 return 0; 501 502 return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val); 503 } 504 EXPORT_SYMBOL(pcie_capability_write_word); 505 506 int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val) 507 { 508 if (pos & 3) 509 return PCIBIOS_BAD_REGISTER_NUMBER; 510 511 if (!pcie_capability_reg_implemented(dev, pos)) 512 return 0; 513 514 return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val); 515 } 516 EXPORT_SYMBOL(pcie_capability_write_dword); 517 518 int pcie_capability_clear_and_set_word_unlocked(struct pci_dev *dev, int pos, 519 u16 clear, u16 set) 520 { 521 int ret; 522 u16 val; 523 524 ret = pcie_capability_read_word(dev, pos, &val); 525 if (ret) 526 return ret; 527 528 val &= ~clear; 529 val |= set; 530 return pcie_capability_write_word(dev, pos, val); 531 } 532 EXPORT_SYMBOL(pcie_capability_clear_and_set_word_unlocked); 533 534 int pcie_capability_clear_and_set_word_locked(struct pci_dev *dev, int pos, 535 u16 clear, u16 set) 536 { 537 unsigned long flags; 538 int ret; 539 540 spin_lock_irqsave(&dev->pcie_cap_lock, flags); 541 ret = pcie_capability_clear_and_set_word_unlocked(dev, pos, clear, set); 542 spin_unlock_irqrestore(&dev->pcie_cap_lock, flags); 543 544 return ret; 545 } 546 EXPORT_SYMBOL(pcie_capability_clear_and_set_word_locked); 547 548 int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos, 549 u32 clear, u32 set) 550 { 551 int ret; 552 u32 val; 553 554 ret = pcie_capability_read_dword(dev, pos, &val); 555 if (ret) 556 return ret; 557 558 val &= ~clear; 559 val |= set; 560 return pcie_capability_write_dword(dev, pos, val); 561 } 562 EXPORT_SYMBOL(pcie_capability_clear_and_set_dword); 563 564 int pci_read_config_byte(const struct pci_dev *dev, int where, u8 *val) 565 { 566 if (pci_dev_is_disconnected(dev)) { 567 PCI_SET_ERROR_RESPONSE(val); 568 return PCIBIOS_DEVICE_NOT_FOUND; 569 } 570 return pci_bus_read_config_byte(dev->bus, dev->devfn, where, val); 571 } 572 EXPORT_SYMBOL(pci_read_config_byte); 573 574 int pci_read_config_word(const struct pci_dev *dev, int where, u16 *val) 575 { 576 if (pci_dev_is_disconnected(dev)) { 577 PCI_SET_ERROR_RESPONSE(val); 578 return PCIBIOS_DEVICE_NOT_FOUND; 579 } 580 return pci_bus_read_config_word(dev->bus, dev->devfn, where, val); 581 } 582 EXPORT_SYMBOL(pci_read_config_word); 583 584 int pci_read_config_dword(const struct pci_dev *dev, int where, 585 u32 *val) 586 { 587 if (pci_dev_is_disconnected(dev)) { 588 PCI_SET_ERROR_RESPONSE(val); 589 return PCIBIOS_DEVICE_NOT_FOUND; 590 } 591 return pci_bus_read_config_dword(dev->bus, dev->devfn, where, val); 592 } 593 EXPORT_SYMBOL(pci_read_config_dword); 594 595 int pci_write_config_byte(const struct pci_dev *dev, int where, u8 val) 596 { 597 if (pci_dev_is_disconnected(dev)) 598 return PCIBIOS_DEVICE_NOT_FOUND; 599 return pci_bus_write_config_byte(dev->bus, dev->devfn, where, val); 600 } 601 EXPORT_SYMBOL(pci_write_config_byte); 602 603 int pci_write_config_word(const struct pci_dev *dev, int where, u16 val) 604 { 605 if (pci_dev_is_disconnected(dev)) 606 return PCIBIOS_DEVICE_NOT_FOUND; 607 return pci_bus_write_config_word(dev->bus, dev->devfn, where, val); 608 } 609 EXPORT_SYMBOL(pci_write_config_word); 610 611 int pci_write_config_dword(const struct pci_dev *dev, int where, 612 u32 val) 613 { 614 if (pci_dev_is_disconnected(dev)) 615 return PCIBIOS_DEVICE_NOT_FOUND; 616 return pci_bus_write_config_dword(dev->bus, dev->devfn, where, val); 617 } 618 EXPORT_SYMBOL(pci_write_config_dword); 619 620 void pci_clear_and_set_config_dword(const struct pci_dev *dev, int pos, 621 u32 clear, u32 set) 622 { 623 u32 val; 624 625 pci_read_config_dword(dev, pos, &val); 626 val &= ~clear; 627 val |= set; 628 pci_write_config_dword(dev, pos, val); 629 } 630 EXPORT_SYMBOL(pci_clear_and_set_config_dword); 631