1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * ARM GIC v2m MSI(-X) support 4 * Support for Message Signaled Interrupts for systems that 5 * implement ARM Generic Interrupt Controller: GICv2m. 6 * 7 * Copyright (C) 2014 Advanced Micro Devices, Inc. 8 * Authors: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> 9 * Harish Kasiviswanathan <harish.kasiviswanathan@amd.com> 10 * Brandon Anderson <brandon.anderson@amd.com> 11 */ 12 13 #define pr_fmt(fmt) "GICv2m: " fmt 14 15 #include <linux/acpi.h> 16 #include <linux/iommu.h> 17 #include <linux/irq.h> 18 #include <linux/irqdomain.h> 19 #include <linux/kernel.h> 20 #include <linux/pci.h> 21 #include <linux/msi.h> 22 #include <linux/of_address.h> 23 #include <linux/of_pci.h> 24 #include <linux/slab.h> 25 #include <linux/spinlock.h> 26 #include <linux/irqchip/arm-gic.h> 27 #include <linux/irqchip/arm-gic-common.h> 28 29 #include <linux/irqchip/irq-msi-lib.h> 30 31 /* 32 * MSI_TYPER: 33 * [31:26] Reserved 34 * [25:16] lowest SPI assigned to MSI 35 * [15:10] Reserved 36 * [9:0] Numer of SPIs assigned to MSI 37 */ 38 #define V2M_MSI_TYPER 0x008 39 #define V2M_MSI_TYPER_BASE_SHIFT 16 40 #define V2M_MSI_TYPER_BASE_MASK 0x3FF 41 #define V2M_MSI_TYPER_NUM_MASK 0x3FF 42 #define V2M_MSI_SETSPI_NS 0x040 43 #define V2M_MIN_SPI 32 44 #define V2M_MAX_SPI 1019 45 #define V2M_MSI_IIDR 0xFCC 46 47 #define V2M_MSI_TYPER_BASE_SPI(x) \ 48 (((x) >> V2M_MSI_TYPER_BASE_SHIFT) & V2M_MSI_TYPER_BASE_MASK) 49 50 #define V2M_MSI_TYPER_NUM_SPI(x) ((x) & V2M_MSI_TYPER_NUM_MASK) 51 52 /* APM X-Gene with GICv2m MSI_IIDR register value */ 53 #define XGENE_GICV2M_MSI_IIDR 0x06000170 54 55 /* Broadcom NS2 GICv2m MSI_IIDR register value */ 56 #define BCM_NS2_GICV2M_MSI_IIDR 0x0000013f 57 58 /* List of flags for specific v2m implementation */ 59 #define GICV2M_NEEDS_SPI_OFFSET 0x00000001 60 #define GICV2M_GRAVITON_ADDRESS_ONLY 0x00000002 61 62 static LIST_HEAD(v2m_nodes); 63 static DEFINE_SPINLOCK(v2m_lock); 64 65 struct v2m_data { 66 struct list_head entry; 67 struct fwnode_handle *fwnode; 68 struct resource res; /* GICv2m resource */ 69 void __iomem *base; /* GICv2m virt address */ 70 u32 spi_start; /* The SPI number that MSIs start */ 71 u32 nr_spis; /* The number of SPIs for MSIs */ 72 u32 spi_offset; /* offset to be subtracted from SPI number */ 73 unsigned long *bm; /* MSI vector bitmap */ 74 u32 flags; /* v2m flags for specific implementation */ 75 }; 76 77 static phys_addr_t gicv2m_get_msi_addr(struct v2m_data *v2m, int hwirq) 78 { 79 if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY) 80 return v2m->res.start | ((hwirq - 32) << 3); 81 else 82 return v2m->res.start + V2M_MSI_SETSPI_NS; 83 } 84 85 static void gicv2m_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) 86 { 87 struct v2m_data *v2m = irq_data_get_irq_chip_data(data); 88 phys_addr_t addr = gicv2m_get_msi_addr(v2m, data->hwirq); 89 90 if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY) 91 msg->data = 0; 92 else 93 msg->data = data->hwirq; 94 if (v2m->flags & GICV2M_NEEDS_SPI_OFFSET) 95 msg->data -= v2m->spi_offset; 96 97 msi_msg_set_addr(irq_data_get_msi_desc(data), msg, addr); 98 } 99 100 static struct irq_chip gicv2m_irq_chip = { 101 .name = "GICv2m", 102 .irq_mask = irq_chip_mask_parent, 103 .irq_unmask = irq_chip_unmask_parent, 104 .irq_eoi = irq_chip_eoi_parent, 105 .irq_set_affinity = irq_chip_set_affinity_parent, 106 .irq_compose_msi_msg = gicv2m_compose_msi_msg, 107 }; 108 109 static int gicv2m_irq_gic_domain_alloc(struct irq_domain *domain, 110 unsigned int virq, 111 irq_hw_number_t hwirq) 112 { 113 struct irq_fwspec fwspec; 114 struct irq_data *d; 115 int err; 116 117 if (is_of_node(domain->parent->fwnode)) { 118 fwspec.fwnode = domain->parent->fwnode; 119 fwspec.param_count = 3; 120 fwspec.param[0] = 0; 121 fwspec.param[1] = hwirq - 32; 122 fwspec.param[2] = IRQ_TYPE_EDGE_RISING; 123 } else if (is_fwnode_irqchip(domain->parent->fwnode)) { 124 fwspec.fwnode = domain->parent->fwnode; 125 fwspec.param_count = 2; 126 fwspec.param[0] = hwirq; 127 fwspec.param[1] = IRQ_TYPE_EDGE_RISING; 128 } else { 129 return -EINVAL; 130 } 131 132 err = irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec); 133 if (err) 134 return err; 135 136 /* Configure the interrupt line to be edge */ 137 d = irq_domain_get_irq_data(domain->parent, virq); 138 d->chip->irq_set_type(d, IRQ_TYPE_EDGE_RISING); 139 return 0; 140 } 141 142 static void gicv2m_unalloc_msi(struct v2m_data *v2m, unsigned int hwirq, 143 int nr_irqs) 144 { 145 spin_lock(&v2m_lock); 146 bitmap_release_region(v2m->bm, hwirq - v2m->spi_start, 147 get_count_order(nr_irqs)); 148 spin_unlock(&v2m_lock); 149 } 150 151 static int gicv2m_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 152 unsigned int nr_irqs, void *args) 153 { 154 msi_alloc_info_t *info = args; 155 struct v2m_data *v2m = NULL, *tmp; 156 int hwirq, i, err = 0; 157 unsigned long offset; 158 unsigned long align_mask = nr_irqs - 1; 159 160 spin_lock(&v2m_lock); 161 list_for_each_entry(tmp, &v2m_nodes, entry) { 162 unsigned long align_off = tmp->spi_start - (tmp->spi_start & ~align_mask); 163 164 offset = bitmap_find_next_zero_area_off(tmp->bm, tmp->nr_spis, 0, 165 nr_irqs, align_mask, align_off); 166 if (offset < tmp->nr_spis) { 167 v2m = tmp; 168 bitmap_set(v2m->bm, offset, nr_irqs); 169 break; 170 } 171 } 172 spin_unlock(&v2m_lock); 173 174 if (!v2m) 175 return -ENOSPC; 176 177 hwirq = v2m->spi_start + offset; 178 179 err = iommu_dma_prepare_msi(info->desc, 180 gicv2m_get_msi_addr(v2m, hwirq)); 181 if (err) 182 return err; 183 184 for (i = 0; i < nr_irqs; i++) { 185 err = gicv2m_irq_gic_domain_alloc(domain, virq + i, hwirq + i); 186 if (err) 187 goto fail; 188 189 irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, 190 &gicv2m_irq_chip, v2m); 191 } 192 193 return 0; 194 195 fail: 196 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 197 gicv2m_unalloc_msi(v2m, hwirq, nr_irqs); 198 return err; 199 } 200 201 static void gicv2m_irq_domain_free(struct irq_domain *domain, 202 unsigned int virq, unsigned int nr_irqs) 203 { 204 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 205 struct v2m_data *v2m = irq_data_get_irq_chip_data(d); 206 207 gicv2m_unalloc_msi(v2m, d->hwirq, nr_irqs); 208 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 209 } 210 211 static const struct irq_domain_ops gicv2m_domain_ops = { 212 .select = msi_lib_irq_domain_select, 213 .alloc = gicv2m_irq_domain_alloc, 214 .free = gicv2m_irq_domain_free, 215 }; 216 217 static bool is_msi_spi_valid(u32 base, u32 num) 218 { 219 if (base < V2M_MIN_SPI) { 220 pr_err("Invalid MSI base SPI (base:%u)\n", base); 221 return false; 222 } 223 224 if ((num == 0) || (base + num > V2M_MAX_SPI)) { 225 pr_err("Number of SPIs (%u) exceed maximum (%u)\n", 226 num, V2M_MAX_SPI - V2M_MIN_SPI + 1); 227 return false; 228 } 229 230 return true; 231 } 232 233 static void __init gicv2m_teardown(void) 234 { 235 struct v2m_data *v2m, *tmp; 236 237 list_for_each_entry_safe(v2m, tmp, &v2m_nodes, entry) { 238 list_del(&v2m->entry); 239 bitmap_free(v2m->bm); 240 iounmap(v2m->base); 241 of_node_put(to_of_node(v2m->fwnode)); 242 if (is_fwnode_irqchip(v2m->fwnode)) 243 irq_domain_free_fwnode(v2m->fwnode); 244 kfree(v2m); 245 } 246 } 247 248 249 #define GICV2M_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ 250 MSI_FLAG_USE_DEF_CHIP_OPS | \ 251 MSI_FLAG_PCI_MSI_MASK_PARENT) 252 253 #define GICV2M_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ 254 MSI_FLAG_PCI_MSIX | \ 255 MSI_FLAG_MULTI_PCI_MSI) 256 257 static struct msi_parent_ops gicv2m_msi_parent_ops = { 258 .supported_flags = GICV2M_MSI_FLAGS_SUPPORTED, 259 .required_flags = GICV2M_MSI_FLAGS_REQUIRED, 260 .chip_flags = MSI_CHIP_FLAG_SET_EOI, 261 .bus_select_token = DOMAIN_BUS_NEXUS, 262 .bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI, 263 .prefix = "GICv2m-", 264 .init_dev_msi_info = msi_lib_init_dev_msi_info, 265 }; 266 267 static __init int gicv2m_allocate_domains(struct irq_domain *parent) 268 { 269 struct irq_domain_info info = { 270 .ops = &gicv2m_domain_ops, 271 .parent = parent, 272 }; 273 struct v2m_data *v2m; 274 275 v2m = list_first_entry_or_null(&v2m_nodes, struct v2m_data, entry); 276 if (!v2m) 277 return 0; 278 279 info.host_data = v2m; 280 info.fwnode = v2m->fwnode; 281 282 if (!msi_create_parent_irq_domain(&info, &gicv2m_msi_parent_ops)) { 283 pr_err("Failed to create GICv2m domain\n"); 284 return -ENOMEM; 285 } 286 return 0; 287 } 288 289 static int __init gicv2m_init_one(struct fwnode_handle *fwnode, 290 u32 spi_start, u32 nr_spis, 291 struct resource *res, u32 flags) 292 { 293 int ret; 294 struct v2m_data *v2m; 295 296 v2m = kzalloc(sizeof(struct v2m_data), GFP_KERNEL); 297 if (!v2m) 298 return -ENOMEM; 299 300 INIT_LIST_HEAD(&v2m->entry); 301 v2m->fwnode = fwnode; 302 v2m->flags = flags; 303 304 memcpy(&v2m->res, res, sizeof(struct resource)); 305 306 v2m->base = ioremap(v2m->res.start, resource_size(&v2m->res)); 307 if (!v2m->base) { 308 pr_err("Failed to map GICv2m resource\n"); 309 ret = -ENOMEM; 310 goto err_free_v2m; 311 } 312 313 if (spi_start && nr_spis) { 314 v2m->spi_start = spi_start; 315 v2m->nr_spis = nr_spis; 316 } else { 317 u32 typer; 318 319 /* Graviton should always have explicit spi_start/nr_spis */ 320 if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY) { 321 ret = -EINVAL; 322 goto err_iounmap; 323 } 324 typer = readl_relaxed(v2m->base + V2M_MSI_TYPER); 325 326 v2m->spi_start = V2M_MSI_TYPER_BASE_SPI(typer); 327 v2m->nr_spis = V2M_MSI_TYPER_NUM_SPI(typer); 328 } 329 330 if (!is_msi_spi_valid(v2m->spi_start, v2m->nr_spis)) { 331 ret = -EINVAL; 332 goto err_iounmap; 333 } 334 335 /* 336 * APM X-Gene GICv2m implementation has an erratum where 337 * the MSI data needs to be the offset from the spi_start 338 * in order to trigger the correct MSI interrupt. This is 339 * different from the standard GICv2m implementation where 340 * the MSI data is the absolute value within the range from 341 * spi_start to (spi_start + num_spis). 342 * 343 * Broadcom NS2 GICv2m implementation has an erratum where the MSI data 344 * is 'spi_number - 32' 345 * 346 * Reading that register fails on the Graviton implementation 347 */ 348 if (!(v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY)) { 349 switch (readl_relaxed(v2m->base + V2M_MSI_IIDR)) { 350 case XGENE_GICV2M_MSI_IIDR: 351 v2m->flags |= GICV2M_NEEDS_SPI_OFFSET; 352 v2m->spi_offset = v2m->spi_start; 353 break; 354 case BCM_NS2_GICV2M_MSI_IIDR: 355 v2m->flags |= GICV2M_NEEDS_SPI_OFFSET; 356 v2m->spi_offset = 32; 357 break; 358 } 359 } 360 v2m->bm = bitmap_zalloc(v2m->nr_spis, GFP_KERNEL); 361 if (!v2m->bm) { 362 ret = -ENOMEM; 363 goto err_iounmap; 364 } 365 366 list_add_tail(&v2m->entry, &v2m_nodes); 367 368 pr_info("range%pR, SPI[%d:%d]\n", res, 369 v2m->spi_start, (v2m->spi_start + v2m->nr_spis - 1)); 370 return 0; 371 372 err_iounmap: 373 iounmap(v2m->base); 374 err_free_v2m: 375 kfree(v2m); 376 return ret; 377 } 378 379 static __initconst struct of_device_id gicv2m_device_id[] = { 380 { .compatible = "arm,gic-v2m-frame", }, 381 {}, 382 }; 383 384 static int __init gicv2m_of_init(struct fwnode_handle *parent_handle, 385 struct irq_domain *parent) 386 { 387 int ret = 0; 388 struct device_node *node = to_of_node(parent_handle); 389 struct device_node *child; 390 391 for (child = of_find_matching_node(node, gicv2m_device_id); child; 392 child = of_find_matching_node(child, gicv2m_device_id)) { 393 u32 spi_start = 0, nr_spis = 0; 394 struct resource res; 395 396 if (!of_property_read_bool(child, "msi-controller")) 397 continue; 398 399 ret = of_address_to_resource(child, 0, &res); 400 if (ret) { 401 pr_err("Failed to allocate v2m resource.\n"); 402 break; 403 } 404 405 if (!of_property_read_u32(child, "arm,msi-base-spi", 406 &spi_start) && 407 !of_property_read_u32(child, "arm,msi-num-spis", &nr_spis)) 408 pr_info("DT overriding V2M MSI_TYPER (base:%u, num:%u)\n", 409 spi_start, nr_spis); 410 411 ret = gicv2m_init_one(&child->fwnode, spi_start, nr_spis, 412 &res, 0); 413 if (ret) 414 break; 415 } 416 417 if (ret && child) 418 of_node_put(child); 419 if (!ret) 420 ret = gicv2m_allocate_domains(parent); 421 if (ret) 422 gicv2m_teardown(); 423 return ret; 424 } 425 426 #ifdef CONFIG_ACPI 427 static int acpi_num_msi; 428 429 static struct fwnode_handle *gicv2m_get_fwnode(struct device *dev) 430 { 431 struct v2m_data *data; 432 433 if (WARN_ON(acpi_num_msi <= 0)) 434 return NULL; 435 436 /* We only return the fwnode of the first MSI frame. */ 437 data = list_first_entry_or_null(&v2m_nodes, struct v2m_data, entry); 438 if (!data) 439 return NULL; 440 441 return data->fwnode; 442 } 443 444 static __init bool acpi_check_amazon_graviton_quirks(void) 445 { 446 static struct acpi_table_madt *madt; 447 acpi_status status; 448 bool rc = false; 449 450 #define ACPI_AMZN_OEM_ID "AMAZON" 451 452 status = acpi_get_table(ACPI_SIG_MADT, 0, 453 (struct acpi_table_header **)&madt); 454 455 if (ACPI_FAILURE(status) || !madt) 456 return rc; 457 rc = !memcmp(madt->header.oem_id, ACPI_AMZN_OEM_ID, ACPI_OEM_ID_SIZE); 458 acpi_put_table((struct acpi_table_header *)madt); 459 460 return rc; 461 } 462 463 static int __init 464 acpi_parse_madt_msi(union acpi_subtable_headers *header, 465 const unsigned long end) 466 { 467 int ret; 468 struct resource res; 469 u32 spi_start = 0, nr_spis = 0; 470 struct acpi_madt_generic_msi_frame *m; 471 struct fwnode_handle *fwnode; 472 u32 flags = 0; 473 474 m = (struct acpi_madt_generic_msi_frame *)header; 475 if (BAD_MADT_ENTRY(m, end)) 476 return -EINVAL; 477 478 res.start = m->base_address; 479 res.end = m->base_address + SZ_4K - 1; 480 res.flags = IORESOURCE_MEM; 481 482 if (acpi_check_amazon_graviton_quirks()) { 483 pr_info("applying Amazon Graviton quirk\n"); 484 res.end = res.start + SZ_8K - 1; 485 flags |= GICV2M_GRAVITON_ADDRESS_ONLY; 486 gicv2m_msi_parent_ops.supported_flags &= ~MSI_FLAG_MULTI_PCI_MSI; 487 } 488 489 if (m->flags & ACPI_MADT_OVERRIDE_SPI_VALUES) { 490 spi_start = m->spi_base; 491 nr_spis = m->spi_count; 492 493 pr_info("ACPI overriding V2M MSI_TYPER (base:%u, num:%u)\n", 494 spi_start, nr_spis); 495 } 496 497 fwnode = irq_domain_alloc_fwnode(&res.start); 498 if (!fwnode) { 499 pr_err("Unable to allocate GICv2m domain token\n"); 500 return -EINVAL; 501 } 502 503 ret = gicv2m_init_one(fwnode, spi_start, nr_spis, &res, flags); 504 if (ret) 505 irq_domain_free_fwnode(fwnode); 506 507 return ret; 508 } 509 510 static int __init gicv2m_acpi_init(struct irq_domain *parent) 511 { 512 int ret; 513 514 if (acpi_num_msi > 0) 515 return 0; 516 517 acpi_num_msi = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_MSI_FRAME, 518 acpi_parse_madt_msi, 0); 519 520 if (acpi_num_msi <= 0) 521 goto err_out; 522 523 ret = gicv2m_allocate_domains(parent); 524 if (ret) 525 goto err_out; 526 527 pci_msi_register_fwnode_provider(&gicv2m_get_fwnode); 528 529 return 0; 530 531 err_out: 532 gicv2m_teardown(); 533 return -EINVAL; 534 } 535 #else /* CONFIG_ACPI */ 536 static int __init gicv2m_acpi_init(struct irq_domain *parent) 537 { 538 return -EINVAL; 539 } 540 #endif /* CONFIG_ACPI */ 541 542 int __init gicv2m_init(struct fwnode_handle *parent_handle, 543 struct irq_domain *parent) 544 { 545 if (is_of_node(parent_handle)) 546 return gicv2m_of_init(parent_handle, parent); 547 548 return gicv2m_acpi_init(parent); 549 } 550