1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Message Signaled Interrupt (MSI) - irqdomain support 4 */ 5 #include <linux/acpi_iort.h> 6 #include <linux/irqdomain.h> 7 #include <linux/of_irq.h> 8 9 #include "msi.h" 10 11 int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) 12 { 13 struct irq_domain *domain; 14 15 domain = dev_get_msi_domain(&dev->dev); 16 if (domain && irq_domain_is_hierarchy(domain)) 17 return msi_domain_alloc_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN, nvec); 18 19 return pci_msi_legacy_setup_msi_irqs(dev, nvec, type); 20 } 21 22 void pci_msi_teardown_msi_irqs(struct pci_dev *dev) 23 { 24 struct irq_domain *domain; 25 26 domain = dev_get_msi_domain(&dev->dev); 27 if (domain && irq_domain_is_hierarchy(domain)) { 28 msi_domain_free_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN); 29 } else { 30 pci_msi_legacy_teardown_msi_irqs(dev); 31 msi_free_msi_descs(&dev->dev); 32 } 33 } 34 35 /** 36 * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space 37 * @irq_data: Pointer to interrupt data of the MSI interrupt 38 * @msg: Pointer to the message 39 */ 40 static void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg) 41 { 42 struct msi_desc *desc = irq_data_get_msi_desc(irq_data); 43 44 /* 45 * For MSI-X desc->irq is always equal to irq_data->irq. For 46 * MSI only the first interrupt of MULTI MSI passes the test. 47 */ 48 if (desc->irq == irq_data->irq) 49 __pci_write_msi_msg(desc, msg); 50 } 51 52 /** 53 * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source 54 * @desc: Pointer to the MSI descriptor 55 * 56 * The ID number is only used within the irqdomain. 57 */ 58 static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc) 59 { 60 struct pci_dev *dev = msi_desc_to_pci_dev(desc); 61 62 return (irq_hw_number_t)desc->msi_index | 63 pci_dev_id(dev) << 11 | 64 ((irq_hw_number_t)(pci_domain_nr(dev->bus) & 0xFFFFFFFF)) << 27; 65 } 66 67 static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, 68 struct msi_desc *desc) 69 { 70 arg->desc = desc; 71 arg->hwirq = pci_msi_domain_calc_hwirq(desc); 72 } 73 74 static struct msi_domain_ops pci_msi_domain_ops_default = { 75 .set_desc = pci_msi_domain_set_desc, 76 }; 77 78 static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) 79 { 80 struct msi_domain_ops *ops = info->ops; 81 82 if (ops == NULL) { 83 info->ops = &pci_msi_domain_ops_default; 84 } else { 85 if (ops->set_desc == NULL) 86 ops->set_desc = pci_msi_domain_set_desc; 87 } 88 } 89 90 static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info) 91 { 92 struct irq_chip *chip = info->chip; 93 94 BUG_ON(!chip); 95 if (!chip->irq_write_msi_msg) 96 chip->irq_write_msi_msg = pci_msi_domain_write_msg; 97 if (!chip->irq_mask) 98 chip->irq_mask = pci_msi_mask_irq; 99 if (!chip->irq_unmask) 100 chip->irq_unmask = pci_msi_unmask_irq; 101 } 102 103 /** 104 * pci_msi_create_irq_domain - Create a MSI interrupt domain 105 * @fwnode: Optional fwnode of the interrupt controller 106 * @info: MSI domain info 107 * @parent: Parent irq domain 108 * 109 * Updates the domain and chip ops and creates a MSI interrupt domain. 110 * 111 * Returns: 112 * A domain pointer or NULL in case of failure. 113 */ 114 struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, 115 struct msi_domain_info *info, 116 struct irq_domain *parent) 117 { 118 if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE)) 119 info->flags &= ~MSI_FLAG_LEVEL_CAPABLE; 120 121 if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) 122 pci_msi_domain_update_dom_ops(info); 123 if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) 124 pci_msi_domain_update_chip_ops(info); 125 126 /* Let the core code free MSI descriptors when freeing interrupts */ 127 info->flags |= MSI_FLAG_FREE_MSI_DESCS; 128 129 info->flags |= MSI_FLAG_ACTIVATE_EARLY | MSI_FLAG_DEV_SYSFS; 130 if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE)) 131 info->flags |= MSI_FLAG_MUST_REACTIVATE; 132 133 /* PCI-MSI is oneshot-safe */ 134 info->chip->flags |= IRQCHIP_ONESHOT_SAFE; 135 /* Let the core update the bus token */ 136 info->bus_token = DOMAIN_BUS_PCI_MSI; 137 138 return msi_create_irq_domain(fwnode, info, parent); 139 } 140 EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain); 141 142 /* 143 * Per device MSI[-X] domain functionality 144 */ 145 static void pci_device_domain_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) 146 { 147 arg->desc = desc; 148 arg->hwirq = desc->msi_index; 149 } 150 151 static void pci_irq_mask_msi(struct irq_data *data) 152 { 153 struct msi_desc *desc = irq_data_get_msi_desc(data); 154 155 pci_msi_mask(desc, BIT(data->irq - desc->irq)); 156 } 157 158 static void pci_irq_unmask_msi(struct irq_data *data) 159 { 160 struct msi_desc *desc = irq_data_get_msi_desc(data); 161 162 pci_msi_unmask(desc, BIT(data->irq - desc->irq)); 163 } 164 165 #ifdef CONFIG_GENERIC_IRQ_RESERVATION_MODE 166 # define MSI_REACTIVATE MSI_FLAG_MUST_REACTIVATE 167 #else 168 # define MSI_REACTIVATE 0 169 #endif 170 171 #define MSI_COMMON_FLAGS (MSI_FLAG_FREE_MSI_DESCS | \ 172 MSI_FLAG_ACTIVATE_EARLY | \ 173 MSI_FLAG_DEV_SYSFS | \ 174 MSI_REACTIVATE) 175 176 static const struct msi_domain_template pci_msi_template = { 177 .chip = { 178 .name = "PCI-MSI", 179 .irq_mask = pci_irq_mask_msi, 180 .irq_unmask = pci_irq_unmask_msi, 181 .irq_write_msi_msg = pci_msi_domain_write_msg, 182 .flags = IRQCHIP_ONESHOT_SAFE, 183 }, 184 185 .ops = { 186 .set_desc = pci_device_domain_set_desc, 187 }, 188 189 .info = { 190 .flags = MSI_COMMON_FLAGS | MSI_FLAG_MULTI_PCI_MSI, 191 .bus_token = DOMAIN_BUS_PCI_DEVICE_MSI, 192 }, 193 }; 194 195 static void pci_irq_mask_msix(struct irq_data *data) 196 { 197 pci_msix_mask(irq_data_get_msi_desc(data)); 198 } 199 200 static void pci_irq_unmask_msix(struct irq_data *data) 201 { 202 pci_msix_unmask(irq_data_get_msi_desc(data)); 203 } 204 205 static void pci_msix_prepare_desc(struct irq_domain *domain, msi_alloc_info_t *arg, 206 struct msi_desc *desc) 207 { 208 /* Don't fiddle with preallocated MSI descriptors */ 209 if (!desc->pci.mask_base) 210 msix_prepare_msi_desc(to_pci_dev(desc->dev), desc); 211 } 212 213 static const struct msi_domain_template pci_msix_template = { 214 .chip = { 215 .name = "PCI-MSIX", 216 .irq_mask = pci_irq_mask_msix, 217 .irq_unmask = pci_irq_unmask_msix, 218 .irq_write_msi_msg = pci_msi_domain_write_msg, 219 .flags = IRQCHIP_ONESHOT_SAFE, 220 }, 221 222 .ops = { 223 .prepare_desc = pci_msix_prepare_desc, 224 .set_desc = pci_device_domain_set_desc, 225 }, 226 227 .info = { 228 .flags = MSI_COMMON_FLAGS | MSI_FLAG_PCI_MSIX | 229 MSI_FLAG_PCI_MSIX_ALLOC_DYN, 230 .bus_token = DOMAIN_BUS_PCI_DEVICE_MSIX, 231 }, 232 }; 233 234 static bool pci_match_device_domain(struct pci_dev *pdev, enum irq_domain_bus_token bus_token) 235 { 236 return msi_match_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN, bus_token); 237 } 238 239 static bool pci_create_device_domain(struct pci_dev *pdev, const struct msi_domain_template *tmpl, 240 unsigned int hwsize) 241 { 242 struct irq_domain *domain = dev_get_msi_domain(&pdev->dev); 243 244 if (!domain || !irq_domain_is_msi_parent(domain)) 245 return true; 246 247 return msi_create_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN, tmpl, 248 hwsize, NULL, NULL); 249 } 250 251 /** 252 * pci_setup_msi_device_domain - Setup a device MSI interrupt domain 253 * @pdev: The PCI device to create the domain on 254 * 255 * Return: 256 * True when: 257 * - The device does not have a MSI parent irq domain associated, 258 * which keeps the legacy architecture specific and the global 259 * PCI/MSI domain models working 260 * - The MSI domain exists already 261 * - The MSI domain was successfully allocated 262 * False when: 263 * - MSI-X is enabled 264 * - The domain creation fails. 265 * 266 * The created MSI domain is preserved until: 267 * - The device is removed 268 * - MSI is disabled and a MSI-X domain is created 269 */ 270 bool pci_setup_msi_device_domain(struct pci_dev *pdev) 271 { 272 if (WARN_ON_ONCE(pdev->msix_enabled)) 273 return false; 274 275 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSI)) 276 return true; 277 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSIX)) 278 msi_remove_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN); 279 280 return pci_create_device_domain(pdev, &pci_msi_template, 1); 281 } 282 283 /** 284 * pci_setup_msix_device_domain - Setup a device MSI-X interrupt domain 285 * @pdev: The PCI device to create the domain on 286 * @hwsize: The size of the MSI-X vector table 287 * 288 * Return: 289 * True when: 290 * - The device does not have a MSI parent irq domain associated, 291 * which keeps the legacy architecture specific and the global 292 * PCI/MSI domain models working 293 * - The MSI-X domain exists already 294 * - The MSI-X domain was successfully allocated 295 * False when: 296 * - MSI is enabled 297 * - The domain creation fails. 298 * 299 * The created MSI-X domain is preserved until: 300 * - The device is removed 301 * - MSI-X is disabled and a MSI domain is created 302 */ 303 bool pci_setup_msix_device_domain(struct pci_dev *pdev, unsigned int hwsize) 304 { 305 if (WARN_ON_ONCE(pdev->msi_enabled)) 306 return false; 307 308 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSIX)) 309 return true; 310 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSI)) 311 msi_remove_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN); 312 313 return pci_create_device_domain(pdev, &pci_msix_template, hwsize); 314 } 315 316 /** 317 * pci_msi_domain_supports - Check for support of a particular feature flag 318 * @pdev: The PCI device to operate on 319 * @feature_mask: The feature mask to check for (full match) 320 * @mode: If ALLOW_LEGACY this grants the feature when there is no irq domain 321 * associated to the device. If DENY_LEGACY the lack of an irq domain 322 * makes the feature unsupported 323 */ 324 bool pci_msi_domain_supports(struct pci_dev *pdev, unsigned int feature_mask, 325 enum support_mode mode) 326 { 327 struct msi_domain_info *info; 328 struct irq_domain *domain; 329 unsigned int supported; 330 331 domain = dev_get_msi_domain(&pdev->dev); 332 333 if (!domain || !irq_domain_is_hierarchy(domain)) 334 return mode == ALLOW_LEGACY; 335 336 if (!irq_domain_is_msi_parent(domain)) { 337 /* 338 * For "global" PCI/MSI interrupt domains the associated 339 * msi_domain_info::flags is the authoritative source of 340 * information. 341 */ 342 info = domain->host_data; 343 supported = info->flags; 344 } else { 345 /* 346 * For MSI parent domains the supported feature set 347 * is available in the parent ops. This makes checks 348 * possible before actually instantiating the 349 * per device domain because the parent is never 350 * expanding the PCI/MSI functionality. 351 */ 352 supported = domain->msi_parent_ops->supported_flags; 353 } 354 355 return (supported & feature_mask) == feature_mask; 356 } 357 358 /* 359 * Users of the generic MSI infrastructure expect a device to have a single ID, 360 * so with DMA aliases we have to pick the least-worst compromise. Devices with 361 * DMA phantom functions tend to still emit MSIs from the real function number, 362 * so we ignore those and only consider topological aliases where either the 363 * alias device or RID appears on a different bus number. We also make the 364 * reasonable assumption that bridges are walked in an upstream direction (so 365 * the last one seen wins), and the much braver assumption that the most likely 366 * case is that of PCI->PCIe so we should always use the alias RID. This echoes 367 * the logic from intel_irq_remapping's set_msi_sid(), which presumably works 368 * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions 369 * for taking ownership all we can really do is close our eyes and hope... 370 */ 371 static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data) 372 { 373 u32 *pa = data; 374 u8 bus = PCI_BUS_NUM(*pa); 375 376 if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus) 377 *pa = alias; 378 379 return 0; 380 } 381 382 /** 383 * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID) 384 * @domain: The interrupt domain 385 * @pdev: The PCI device. 386 * 387 * The RID for a device is formed from the alias, with a firmware 388 * supplied mapping applied 389 * 390 * Returns: The RID. 391 */ 392 u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev) 393 { 394 struct device_node *of_node; 395 u32 rid = pci_dev_id(pdev); 396 397 pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); 398 399 of_node = irq_domain_get_of_node(domain); 400 rid = of_node ? of_msi_map_id(&pdev->dev, of_node, rid) : 401 iort_msi_map_id(&pdev->dev, rid); 402 403 return rid; 404 } 405 406 /** 407 * pci_msi_get_device_domain - Get the MSI domain for a given PCI device 408 * @pdev: The PCI device 409 * 410 * Use the firmware data to find a device-specific MSI domain 411 * (i.e. not one that is set as a default). 412 * 413 * Returns: The corresponding MSI domain or NULL if none has been found. 414 */ 415 struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev) 416 { 417 struct irq_domain *dom; 418 u32 rid = pci_dev_id(pdev); 419 420 pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid); 421 dom = of_msi_map_get_device_domain(&pdev->dev, rid, DOMAIN_BUS_PCI_MSI); 422 if (!dom) 423 dom = iort_get_device_domain(&pdev->dev, rid, 424 DOMAIN_BUS_PCI_MSI); 425 return dom; 426 } 427