1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Message Signaled Interrupt (MSI) 4 * 5 * Copyright (C) 2003-2004 Intel 6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) 7 * Copyright (C) 2016 Christoph Hellwig. 8 */ 9 #include <linux/bitfield.h> 10 #include <linux/err.h> 11 #include <linux/export.h> 12 #include <linux/irq.h> 13 #include <linux/irqdomain.h> 14 15 #include "../pci.h" 16 #include "msi.h" 17 18 int pci_msi_enable = 1; 19 20 /** 21 * pci_msi_supported - check whether MSI may be enabled on a device 22 * @dev: pointer to the pci_dev data structure of MSI device function 23 * @nvec: how many MSIs have been requested? 24 * 25 * Look at global flags, the device itself, and its parent buses 26 * to determine if MSI/-X are supported for the device. If MSI/-X is 27 * supported return 1, else return 0. 28 **/ 29 static int pci_msi_supported(struct pci_dev *dev, int nvec) 30 { 31 struct pci_bus *bus; 32 33 /* MSI must be globally enabled and supported by the device */ 34 if (!pci_msi_enable) 35 return 0; 36 37 if (!dev || dev->no_msi) 38 return 0; 39 40 /* 41 * You can't ask to have 0 or less MSIs configured. 42 * a) it's stupid .. 43 * b) the list manipulation code assumes nvec >= 1. 44 */ 45 if (nvec < 1) 46 return 0; 47 48 /* 49 * Any bridge which does NOT route MSI transactions from its 50 * secondary bus to its primary bus must set NO_MSI flag on 51 * the secondary pci_bus. 52 * 53 * The NO_MSI flag can either be set directly by: 54 * - arch-specific PCI host bus controller drivers (deprecated) 55 * - quirks for specific PCI bridges 56 * 57 * or indirectly by platform-specific PCI host bridge drivers by 58 * advertising the 'msi_domain' property, which results in 59 * the NO_MSI flag when no MSI domain is found for this bridge 60 * at probe time. 61 */ 62 for (bus = dev->bus; bus; bus = bus->parent) 63 if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) 64 return 0; 65 66 return 1; 67 } 68 69 static void pcim_msi_release(void *pcidev) 70 { 71 struct pci_dev *dev = pcidev; 72 73 dev->is_msi_managed = false; 74 pci_free_irq_vectors(dev); 75 } 76 77 /* 78 * Needs to be separate from pcim_release to prevent an ordering problem 79 * vs. msi_device_data_release() in the MSI core code. 80 */ 81 static int pcim_setup_msi_release(struct pci_dev *dev) 82 { 83 int ret; 84 85 if (!pci_is_managed(dev) || dev->is_msi_managed) 86 return 0; 87 88 ret = devm_add_action(&dev->dev, pcim_msi_release, dev); 89 if (ret) 90 return ret; 91 92 dev->is_msi_managed = true; 93 return 0; 94 } 95 96 /* 97 * Ordering vs. devres: msi device data has to be installed first so that 98 * pcim_msi_release() is invoked before it on device release. 99 */ 100 static int pci_setup_msi_context(struct pci_dev *dev) 101 { 102 int ret = msi_setup_device_data(&dev->dev); 103 104 if (ret) 105 return ret; 106 107 return pcim_setup_msi_release(dev); 108 } 109 110 /* 111 * Helper functions for mask/unmask and MSI message handling 112 */ 113 114 void pci_msi_update_mask(struct msi_desc *desc, u32 clear, u32 set) 115 { 116 raw_spinlock_t *lock = &to_pci_dev(desc->dev)->msi_lock; 117 unsigned long flags; 118 119 if (!desc->pci.msi_attrib.can_mask) 120 return; 121 122 raw_spin_lock_irqsave(lock, flags); 123 desc->pci.msi_mask &= ~clear; 124 desc->pci.msi_mask |= set; 125 pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->pci.mask_pos, 126 desc->pci.msi_mask); 127 raw_spin_unlock_irqrestore(lock, flags); 128 } 129 130 /** 131 * pci_msi_mask_irq - Generic IRQ chip callback to mask PCI/MSI interrupts 132 * @data: pointer to irqdata associated to that interrupt 133 */ 134 void pci_msi_mask_irq(struct irq_data *data) 135 { 136 struct msi_desc *desc = irq_data_get_msi_desc(data); 137 138 __pci_msi_mask_desc(desc, BIT(data->irq - desc->irq)); 139 } 140 EXPORT_SYMBOL_GPL(pci_msi_mask_irq); 141 142 /** 143 * pci_msi_unmask_irq - Generic IRQ chip callback to unmask PCI/MSI interrupts 144 * @data: pointer to irqdata associated to that interrupt 145 */ 146 void pci_msi_unmask_irq(struct irq_data *data) 147 { 148 struct msi_desc *desc = irq_data_get_msi_desc(data); 149 150 __pci_msi_unmask_desc(desc, BIT(data->irq - desc->irq)); 151 } 152 EXPORT_SYMBOL_GPL(pci_msi_unmask_irq); 153 154 void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) 155 { 156 struct pci_dev *dev = msi_desc_to_pci_dev(entry); 157 158 BUG_ON(dev->current_state != PCI_D0); 159 160 if (entry->pci.msi_attrib.is_msix) { 161 void __iomem *base = pci_msix_desc_addr(entry); 162 163 if (WARN_ON_ONCE(entry->pci.msi_attrib.is_virtual)) 164 return; 165 166 msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); 167 msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); 168 msg->data = readl(base + PCI_MSIX_ENTRY_DATA); 169 } else { 170 int pos = dev->msi_cap; 171 u16 data; 172 173 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, 174 &msg->address_lo); 175 if (entry->pci.msi_attrib.is_64) { 176 pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, 177 &msg->address_hi); 178 pci_read_config_word(dev, pos + PCI_MSI_DATA_64, &data); 179 } else { 180 msg->address_hi = 0; 181 pci_read_config_word(dev, pos + PCI_MSI_DATA_32, &data); 182 } 183 msg->data = data; 184 } 185 } 186 187 static inline void pci_write_msg_msi(struct pci_dev *dev, struct msi_desc *desc, 188 struct msi_msg *msg) 189 { 190 int pos = dev->msi_cap; 191 u16 msgctl; 192 193 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); 194 msgctl &= ~PCI_MSI_FLAGS_QSIZE; 195 msgctl |= FIELD_PREP(PCI_MSI_FLAGS_QSIZE, desc->pci.msi_attrib.multiple); 196 pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl); 197 198 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, msg->address_lo); 199 if (desc->pci.msi_attrib.is_64) { 200 pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, msg->address_hi); 201 pci_write_config_word(dev, pos + PCI_MSI_DATA_64, msg->data); 202 } else { 203 pci_write_config_word(dev, pos + PCI_MSI_DATA_32, msg->data); 204 } 205 /* Ensure that the writes are visible in the device */ 206 pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); 207 } 208 209 static inline void pci_write_msg_msix(struct msi_desc *desc, struct msi_msg *msg) 210 { 211 void __iomem *base = pci_msix_desc_addr(desc); 212 u32 ctrl = desc->pci.msix_ctrl; 213 bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); 214 215 if (desc->pci.msi_attrib.is_virtual) 216 return; 217 /* 218 * The specification mandates that the entry is masked 219 * when the message is modified: 220 * 221 * "If software changes the Address or Data value of an 222 * entry while the entry is unmasked, the result is 223 * undefined." 224 */ 225 if (unmasked) 226 pci_msix_write_vector_ctrl(desc, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT); 227 228 writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); 229 writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); 230 writel(msg->data, base + PCI_MSIX_ENTRY_DATA); 231 232 if (unmasked) 233 pci_msix_write_vector_ctrl(desc, ctrl); 234 235 /* Ensure that the writes are visible in the device */ 236 readl(base + PCI_MSIX_ENTRY_DATA); 237 } 238 239 void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) 240 { 241 struct pci_dev *dev = msi_desc_to_pci_dev(entry); 242 243 if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) { 244 /* Don't touch the hardware now */ 245 } else if (entry->pci.msi_attrib.is_msix) { 246 pci_write_msg_msix(entry, msg); 247 } else { 248 pci_write_msg_msi(dev, entry, msg); 249 } 250 251 entry->msg = *msg; 252 253 if (entry->write_msi_msg) 254 entry->write_msi_msg(entry, entry->write_msi_msg_data); 255 } 256 257 void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg) 258 { 259 struct msi_desc *entry = irq_get_msi_desc(irq); 260 261 __pci_write_msi_msg(entry, msg); 262 } 263 EXPORT_SYMBOL_GPL(pci_write_msi_msg); 264 265 266 /* PCI/MSI specific functionality */ 267 268 static void pci_intx_for_msi(struct pci_dev *dev, int enable) 269 { 270 if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG)) 271 pci_intx(dev, enable); 272 } 273 274 static void pci_msi_set_enable(struct pci_dev *dev, int enable) 275 { 276 u16 control; 277 278 pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); 279 control &= ~PCI_MSI_FLAGS_ENABLE; 280 if (enable) 281 control |= PCI_MSI_FLAGS_ENABLE; 282 pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); 283 } 284 285 static int msi_setup_msi_desc(struct pci_dev *dev, int nvec, 286 struct irq_affinity_desc *masks) 287 { 288 struct msi_desc desc; 289 u16 control; 290 291 /* MSI Entry Initialization */ 292 memset(&desc, 0, sizeof(desc)); 293 294 pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); 295 /* Lies, damned lies, and MSIs */ 296 if (dev->dev_flags & PCI_DEV_FLAGS_HAS_MSI_MASKING) 297 control |= PCI_MSI_FLAGS_MASKBIT; 298 if (pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) 299 control &= ~PCI_MSI_FLAGS_MASKBIT; 300 301 desc.nvec_used = nvec; 302 desc.pci.msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT); 303 desc.pci.msi_attrib.can_mask = !!(control & PCI_MSI_FLAGS_MASKBIT); 304 desc.pci.msi_attrib.default_irq = dev->irq; 305 desc.pci.msi_attrib.multi_cap = FIELD_GET(PCI_MSI_FLAGS_QMASK, control); 306 desc.pci.msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec)); 307 desc.affinity = masks; 308 309 if (control & PCI_MSI_FLAGS_64BIT) 310 desc.pci.mask_pos = dev->msi_cap + PCI_MSI_MASK_64; 311 else 312 desc.pci.mask_pos = dev->msi_cap + PCI_MSI_MASK_32; 313 314 /* Save the initial mask status */ 315 if (desc.pci.msi_attrib.can_mask) 316 pci_read_config_dword(dev, desc.pci.mask_pos, &desc.pci.msi_mask); 317 318 return msi_insert_msi_desc(&dev->dev, &desc); 319 } 320 321 static int msi_verify_entries(struct pci_dev *dev) 322 { 323 struct msi_desc *entry; 324 325 if (!dev->no_64bit_msi) 326 return 0; 327 328 msi_for_each_desc(entry, &dev->dev, MSI_DESC_ALL) { 329 if (entry->msg.address_hi) { 330 pci_err(dev, "arch assigned 64-bit MSI address %#x%08x but device only supports 32 bits\n", 331 entry->msg.address_hi, entry->msg.address_lo); 332 break; 333 } 334 } 335 return !entry ? 0 : -EIO; 336 } 337 338 /** 339 * msi_capability_init - configure device's MSI capability structure 340 * @dev: pointer to the pci_dev data structure of MSI device function 341 * @nvec: number of interrupts to allocate 342 * @affd: description of automatic IRQ affinity assignments (may be %NULL) 343 * 344 * Setup the MSI capability structure of the device with the requested 345 * number of interrupts. A return value of zero indicates the successful 346 * setup of an entry with the new MSI IRQ. A negative return value indicates 347 * an error, and a positive return value indicates the number of interrupts 348 * which could have been allocated. 349 */ 350 static int msi_capability_init(struct pci_dev *dev, int nvec, 351 struct irq_affinity *affd) 352 { 353 struct irq_affinity_desc *masks = NULL; 354 struct msi_desc *entry, desc; 355 int ret; 356 357 /* Reject multi-MSI early on irq domain enabled architectures */ 358 if (nvec > 1 && !pci_msi_domain_supports(dev, MSI_FLAG_MULTI_PCI_MSI, ALLOW_LEGACY)) 359 return 1; 360 361 /* 362 * Disable MSI during setup in the hardware, but mark it enabled 363 * so that setup code can evaluate it. 364 */ 365 pci_msi_set_enable(dev, 0); 366 dev->msi_enabled = 1; 367 368 if (affd) 369 masks = irq_create_affinity_masks(nvec, affd); 370 371 msi_lock_descs(&dev->dev); 372 ret = msi_setup_msi_desc(dev, nvec, masks); 373 if (ret) 374 goto fail; 375 376 /* All MSIs are unmasked by default; mask them all */ 377 entry = msi_first_desc(&dev->dev, MSI_DESC_ALL); 378 pci_msi_mask(entry, msi_multi_mask(entry)); 379 /* 380 * Copy the MSI descriptor for the error path because 381 * pci_msi_setup_msi_irqs() will free it for the hierarchical 382 * interrupt domain case. 383 */ 384 memcpy(&desc, entry, sizeof(desc)); 385 386 /* Configure MSI capability structure */ 387 ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI); 388 if (ret) 389 goto err; 390 391 ret = msi_verify_entries(dev); 392 if (ret) 393 goto err; 394 395 /* Set MSI enabled bits */ 396 pci_intx_for_msi(dev, 0); 397 pci_msi_set_enable(dev, 1); 398 399 pcibios_free_irq(dev); 400 dev->irq = entry->irq; 401 goto unlock; 402 403 err: 404 pci_msi_unmask(&desc, msi_multi_mask(&desc)); 405 pci_free_msi_irqs(dev); 406 fail: 407 dev->msi_enabled = 0; 408 unlock: 409 msi_unlock_descs(&dev->dev); 410 kfree(masks); 411 return ret; 412 } 413 414 int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, 415 struct irq_affinity *affd) 416 { 417 int nvec; 418 int rc; 419 420 if (!pci_msi_supported(dev, minvec) || dev->current_state != PCI_D0) 421 return -EINVAL; 422 423 /* Check whether driver already requested MSI-X IRQs */ 424 if (dev->msix_enabled) { 425 pci_info(dev, "can't enable MSI (MSI-X already enabled)\n"); 426 return -EINVAL; 427 } 428 429 if (maxvec < minvec) 430 return -ERANGE; 431 432 if (WARN_ON_ONCE(dev->msi_enabled)) 433 return -EINVAL; 434 435 /* Test for the availability of MSI support */ 436 if (!pci_msi_domain_supports(dev, 0, ALLOW_LEGACY)) 437 return -ENOTSUPP; 438 439 nvec = pci_msi_vec_count(dev); 440 if (nvec < 0) 441 return nvec; 442 if (nvec < minvec) 443 return -ENOSPC; 444 445 if (nvec > maxvec) 446 nvec = maxvec; 447 448 rc = pci_setup_msi_context(dev); 449 if (rc) 450 return rc; 451 452 if (!pci_setup_msi_device_domain(dev)) 453 return -ENODEV; 454 455 for (;;) { 456 if (affd) { 457 nvec = irq_calc_affinity_vectors(minvec, nvec, affd); 458 if (nvec < minvec) 459 return -ENOSPC; 460 } 461 462 rc = msi_capability_init(dev, nvec, affd); 463 if (rc == 0) 464 return nvec; 465 466 if (rc < 0) 467 return rc; 468 if (rc < minvec) 469 return -ENOSPC; 470 471 nvec = rc; 472 } 473 } 474 475 /** 476 * pci_msi_vec_count - Return the number of MSI vectors a device can send 477 * @dev: device to report about 478 * 479 * This function returns the number of MSI vectors a device requested via 480 * Multiple Message Capable register. It returns a negative errno if the 481 * device is not capable sending MSI interrupts. Otherwise, the call succeeds 482 * and returns a power of two, up to a maximum of 2^5 (32), according to the 483 * MSI specification. 484 **/ 485 int pci_msi_vec_count(struct pci_dev *dev) 486 { 487 int ret; 488 u16 msgctl; 489 490 if (!dev->msi_cap) 491 return -EINVAL; 492 493 pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &msgctl); 494 ret = 1 << FIELD_GET(PCI_MSI_FLAGS_QMASK, msgctl); 495 496 return ret; 497 } 498 EXPORT_SYMBOL(pci_msi_vec_count); 499 500 /* 501 * Architecture override returns true when the PCI MSI message should be 502 * written by the generic restore function. 503 */ 504 bool __weak arch_restore_msi_irqs(struct pci_dev *dev) 505 { 506 return true; 507 } 508 509 void __pci_restore_msi_state(struct pci_dev *dev) 510 { 511 struct msi_desc *entry; 512 u16 control; 513 514 if (!dev->msi_enabled) 515 return; 516 517 entry = irq_get_msi_desc(dev->irq); 518 519 pci_intx_for_msi(dev, 0); 520 pci_msi_set_enable(dev, 0); 521 if (arch_restore_msi_irqs(dev)) 522 __pci_write_msi_msg(entry, &entry->msg); 523 524 pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); 525 pci_msi_update_mask(entry, 0, 0); 526 control &= ~PCI_MSI_FLAGS_QSIZE; 527 control |= PCI_MSI_FLAGS_ENABLE | 528 FIELD_PREP(PCI_MSI_FLAGS_QSIZE, entry->pci.msi_attrib.multiple); 529 pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); 530 } 531 532 void pci_msi_shutdown(struct pci_dev *dev) 533 { 534 struct msi_desc *desc; 535 536 if (!pci_msi_enable || !dev || !dev->msi_enabled) 537 return; 538 539 pci_msi_set_enable(dev, 0); 540 pci_intx_for_msi(dev, 1); 541 dev->msi_enabled = 0; 542 543 /* Return the device with MSI unmasked as initial states */ 544 desc = msi_first_desc(&dev->dev, MSI_DESC_ALL); 545 if (!WARN_ON_ONCE(!desc)) 546 pci_msi_unmask(desc, msi_multi_mask(desc)); 547 548 /* Restore dev->irq to its default pin-assertion IRQ */ 549 dev->irq = desc->pci.msi_attrib.default_irq; 550 pcibios_alloc_irq(dev); 551 } 552 553 /* PCI/MSI-X specific functionality */ 554 555 static void pci_msix_clear_and_set_ctrl(struct pci_dev *dev, u16 clear, u16 set) 556 { 557 u16 ctrl; 558 559 pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &ctrl); 560 ctrl &= ~clear; 561 ctrl |= set; 562 pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, ctrl); 563 } 564 565 static void __iomem *msix_map_region(struct pci_dev *dev, 566 unsigned int nr_entries) 567 { 568 resource_size_t phys_addr; 569 u32 table_offset; 570 unsigned long flags; 571 u8 bir; 572 573 pci_read_config_dword(dev, dev->msix_cap + PCI_MSIX_TABLE, 574 &table_offset); 575 bir = (u8)(table_offset & PCI_MSIX_TABLE_BIR); 576 flags = pci_resource_flags(dev, bir); 577 if (!flags || (flags & IORESOURCE_UNSET)) 578 return NULL; 579 580 table_offset &= PCI_MSIX_TABLE_OFFSET; 581 phys_addr = pci_resource_start(dev, bir) + table_offset; 582 583 return ioremap(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); 584 } 585 586 /** 587 * msix_prepare_msi_desc - Prepare a half initialized MSI descriptor for operation 588 * @dev: The PCI device for which the descriptor is prepared 589 * @desc: The MSI descriptor for preparation 590 * 591 * This is separate from msix_setup_msi_descs() below to handle dynamic 592 * allocations for MSI-X after initial enablement. 593 * 594 * Ideally the whole MSI-X setup would work that way, but there is no way to 595 * support this for the legacy arch_setup_msi_irqs() mechanism and for the 596 * fake irq domains like the x86 XEN one. Sigh... 597 * 598 * The descriptor is zeroed and only @desc::msi_index and @desc::affinity 599 * are set. When called from msix_setup_msi_descs() then the is_virtual 600 * attribute is initialized as well. 601 * 602 * Fill in the rest. 603 */ 604 void msix_prepare_msi_desc(struct pci_dev *dev, struct msi_desc *desc) 605 { 606 desc->nvec_used = 1; 607 desc->pci.msi_attrib.is_msix = 1; 608 desc->pci.msi_attrib.is_64 = 1; 609 desc->pci.msi_attrib.default_irq = dev->irq; 610 desc->pci.mask_base = dev->msix_base; 611 612 613 if (!pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY) && 614 !desc->pci.msi_attrib.is_virtual) { 615 void __iomem *addr = pci_msix_desc_addr(desc); 616 617 desc->pci.msi_attrib.can_mask = 1; 618 /* Workaround for SUN NIU insanity, which requires write before read */ 619 if (dev->dev_flags & PCI_DEV_FLAGS_MSIX_TOUCH_ENTRY_DATA_FIRST) 620 writel(0, addr + PCI_MSIX_ENTRY_DATA); 621 desc->pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); 622 } 623 } 624 625 static int msix_setup_msi_descs(struct pci_dev *dev, struct msix_entry *entries, 626 int nvec, struct irq_affinity_desc *masks) 627 { 628 int ret = 0, i, vec_count = pci_msix_vec_count(dev); 629 struct irq_affinity_desc *curmsk; 630 struct msi_desc desc; 631 632 memset(&desc, 0, sizeof(desc)); 633 634 for (i = 0, curmsk = masks; i < nvec; i++, curmsk++) { 635 desc.msi_index = entries ? entries[i].entry : i; 636 desc.affinity = masks ? curmsk : NULL; 637 desc.pci.msi_attrib.is_virtual = desc.msi_index >= vec_count; 638 639 msix_prepare_msi_desc(dev, &desc); 640 641 ret = msi_insert_msi_desc(&dev->dev, &desc); 642 if (ret) 643 break; 644 } 645 return ret; 646 } 647 648 static void msix_update_entries(struct pci_dev *dev, struct msix_entry *entries) 649 { 650 struct msi_desc *desc; 651 652 if (entries) { 653 msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) { 654 entries->vector = desc->irq; 655 entries++; 656 } 657 } 658 } 659 660 static void msix_mask_all(void __iomem *base, int tsize) 661 { 662 u32 ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT; 663 int i; 664 665 for (i = 0; i < tsize; i++, base += PCI_MSIX_ENTRY_SIZE) 666 writel(ctrl, base + PCI_MSIX_ENTRY_VECTOR_CTRL); 667 } 668 669 static int msix_setup_interrupts(struct pci_dev *dev, struct msix_entry *entries, 670 int nvec, struct irq_affinity *affd) 671 { 672 struct irq_affinity_desc *masks = NULL; 673 int ret; 674 675 if (affd) 676 masks = irq_create_affinity_masks(nvec, affd); 677 678 msi_lock_descs(&dev->dev); 679 ret = msix_setup_msi_descs(dev, entries, nvec, masks); 680 if (ret) 681 goto out_free; 682 683 ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX); 684 if (ret) 685 goto out_free; 686 687 /* Check if all MSI entries honor device restrictions */ 688 ret = msi_verify_entries(dev); 689 if (ret) 690 goto out_free; 691 692 msix_update_entries(dev, entries); 693 goto out_unlock; 694 695 out_free: 696 pci_free_msi_irqs(dev); 697 out_unlock: 698 msi_unlock_descs(&dev->dev); 699 kfree(masks); 700 return ret; 701 } 702 703 /** 704 * msix_capability_init - configure device's MSI-X capability 705 * @dev: pointer to the pci_dev data structure of MSI-X device function 706 * @entries: pointer to an array of struct msix_entry entries 707 * @nvec: number of @entries 708 * @affd: Optional pointer to enable automatic affinity assignment 709 * 710 * Setup the MSI-X capability structure of device function with a 711 * single MSI-X IRQ. A return of zero indicates the successful setup of 712 * requested MSI-X entries with allocated IRQs or non-zero for otherwise. 713 **/ 714 static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, 715 int nvec, struct irq_affinity *affd) 716 { 717 int ret, tsize; 718 u16 control; 719 720 /* 721 * Some devices require MSI-X to be enabled before the MSI-X 722 * registers can be accessed. Mask all the vectors to prevent 723 * interrupts coming in before they're fully set up. 724 */ 725 pci_msix_clear_and_set_ctrl(dev, 0, PCI_MSIX_FLAGS_MASKALL | 726 PCI_MSIX_FLAGS_ENABLE); 727 728 /* Mark it enabled so setup functions can query it */ 729 dev->msix_enabled = 1; 730 731 pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); 732 /* Request & Map MSI-X table region */ 733 tsize = msix_table_size(control); 734 dev->msix_base = msix_map_region(dev, tsize); 735 if (!dev->msix_base) { 736 ret = -ENOMEM; 737 goto out_disable; 738 } 739 740 ret = msix_setup_interrupts(dev, entries, nvec, affd); 741 if (ret) 742 goto out_disable; 743 744 /* Disable INTX */ 745 pci_intx_for_msi(dev, 0); 746 747 if (!pci_msi_domain_supports(dev, MSI_FLAG_NO_MASK, DENY_LEGACY)) { 748 /* 749 * Ensure that all table entries are masked to prevent 750 * stale entries from firing in a crash kernel. 751 * 752 * Done late to deal with a broken Marvell NVME device 753 * which takes the MSI-X mask bits into account even 754 * when MSI-X is disabled, which prevents MSI delivery. 755 */ 756 msix_mask_all(dev->msix_base, tsize); 757 } 758 pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); 759 760 pcibios_free_irq(dev); 761 return 0; 762 763 out_disable: 764 dev->msix_enabled = 0; 765 pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE, 0); 766 767 return ret; 768 } 769 770 static bool pci_msix_validate_entries(struct pci_dev *dev, struct msix_entry *entries, int nvec) 771 { 772 bool nogap; 773 int i, j; 774 775 if (!entries) 776 return true; 777 778 nogap = pci_msi_domain_supports(dev, MSI_FLAG_MSIX_CONTIGUOUS, DENY_LEGACY); 779 780 for (i = 0; i < nvec; i++) { 781 /* Check for duplicate entries */ 782 for (j = i + 1; j < nvec; j++) { 783 if (entries[i].entry == entries[j].entry) 784 return false; 785 } 786 /* Check for unsupported gaps */ 787 if (nogap && entries[i].entry != i) 788 return false; 789 } 790 return true; 791 } 792 793 int __pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, int minvec, 794 int maxvec, struct irq_affinity *affd, int flags) 795 { 796 int hwsize, rc, nvec = maxvec; 797 798 if (maxvec < minvec) 799 return -ERANGE; 800 801 if (dev->msi_enabled) { 802 pci_info(dev, "can't enable MSI-X (MSI already enabled)\n"); 803 return -EINVAL; 804 } 805 806 if (WARN_ON_ONCE(dev->msix_enabled)) 807 return -EINVAL; 808 809 /* Check MSI-X early on irq domain enabled architectures */ 810 if (!pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX, ALLOW_LEGACY)) 811 return -ENOTSUPP; 812 813 if (!pci_msi_supported(dev, nvec) || dev->current_state != PCI_D0) 814 return -EINVAL; 815 816 hwsize = pci_msix_vec_count(dev); 817 if (hwsize < 0) 818 return hwsize; 819 820 if (!pci_msix_validate_entries(dev, entries, nvec)) 821 return -EINVAL; 822 823 if (hwsize < nvec) { 824 /* Keep the IRQ virtual hackery working */ 825 if (flags & PCI_IRQ_VIRTUAL) 826 hwsize = nvec; 827 else 828 nvec = hwsize; 829 } 830 831 if (nvec < minvec) 832 return -ENOSPC; 833 834 rc = pci_setup_msi_context(dev); 835 if (rc) 836 return rc; 837 838 if (!pci_setup_msix_device_domain(dev, hwsize)) 839 return -ENODEV; 840 841 for (;;) { 842 if (affd) { 843 nvec = irq_calc_affinity_vectors(minvec, nvec, affd); 844 if (nvec < minvec) 845 return -ENOSPC; 846 } 847 848 rc = msix_capability_init(dev, entries, nvec, affd); 849 if (rc == 0) 850 return nvec; 851 852 if (rc < 0) 853 return rc; 854 if (rc < minvec) 855 return -ENOSPC; 856 857 nvec = rc; 858 } 859 } 860 861 void __pci_restore_msix_state(struct pci_dev *dev) 862 { 863 struct msi_desc *entry; 864 bool write_msg; 865 866 if (!dev->msix_enabled) 867 return; 868 869 /* route the table */ 870 pci_intx_for_msi(dev, 0); 871 pci_msix_clear_and_set_ctrl(dev, 0, 872 PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL); 873 874 write_msg = arch_restore_msi_irqs(dev); 875 876 msi_lock_descs(&dev->dev); 877 msi_for_each_desc(entry, &dev->dev, MSI_DESC_ALL) { 878 if (write_msg) 879 __pci_write_msi_msg(entry, &entry->msg); 880 pci_msix_write_vector_ctrl(entry, entry->pci.msix_ctrl); 881 } 882 msi_unlock_descs(&dev->dev); 883 884 pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); 885 } 886 887 void pci_msix_shutdown(struct pci_dev *dev) 888 { 889 struct msi_desc *desc; 890 891 if (!pci_msi_enable || !dev || !dev->msix_enabled) 892 return; 893 894 if (pci_dev_is_disconnected(dev)) { 895 dev->msix_enabled = 0; 896 return; 897 } 898 899 /* Return the device with MSI-X masked as initial states */ 900 msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) 901 pci_msix_mask(desc); 902 903 pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); 904 pci_intx_for_msi(dev, 1); 905 dev->msix_enabled = 0; 906 pcibios_alloc_irq(dev); 907 } 908 909 /* Common interfaces */ 910 911 void pci_free_msi_irqs(struct pci_dev *dev) 912 { 913 pci_msi_teardown_msi_irqs(dev); 914 915 if (dev->msix_base) { 916 iounmap(dev->msix_base); 917 dev->msix_base = NULL; 918 } 919 } 920 921 /* Misc. infrastructure */ 922 923 struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc) 924 { 925 return to_pci_dev(desc->dev); 926 } 927 EXPORT_SYMBOL(msi_desc_to_pci_dev); 928 929 void pci_no_msi(void) 930 { 931 pci_msi_enable = 0; 932 } 933