1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2022 Ventana Micro Systems Inc. 5 */ 6 7 #define pr_fmt(fmt) "riscv-imsic: " fmt 8 #include <linux/acpi.h> 9 #include <linux/bitmap.h> 10 #include <linux/cpu.h> 11 #include <linux/interrupt.h> 12 #include <linux/io.h> 13 #include <linux/irq.h> 14 #include <linux/irqchip.h> 15 #include <linux/irqdomain.h> 16 #include <linux/module.h> 17 #include <linux/msi.h> 18 #include <linux/pci.h> 19 #include <linux/platform_device.h> 20 #include <linux/spinlock.h> 21 #include <linux/smp.h> 22 23 #include "irq-riscv-imsic-state.h" 24 25 static bool imsic_cpu_page_phys(unsigned int cpu, unsigned int guest_index, 26 phys_addr_t *out_msi_pa) 27 { 28 struct imsic_global_config *global; 29 struct imsic_local_config *local; 30 31 global = &imsic->global; 32 local = per_cpu_ptr(global->local, cpu); 33 34 if (BIT(global->guest_index_bits) <= guest_index) 35 return false; 36 37 if (out_msi_pa) 38 *out_msi_pa = local->msi_pa + (guest_index * IMSIC_MMIO_PAGE_SZ); 39 40 return true; 41 } 42 43 static void imsic_irq_mask(struct irq_data *d) 44 { 45 imsic_vector_mask(irq_data_get_irq_chip_data(d)); 46 } 47 48 static void imsic_irq_unmask(struct irq_data *d) 49 { 50 imsic_vector_unmask(irq_data_get_irq_chip_data(d)); 51 } 52 53 static int imsic_irq_retrigger(struct irq_data *d) 54 { 55 struct imsic_vector *vec = irq_data_get_irq_chip_data(d); 56 struct imsic_local_config *local; 57 58 if (WARN_ON(!vec)) 59 return -ENOENT; 60 61 local = per_cpu_ptr(imsic->global.local, vec->cpu); 62 writel_relaxed(vec->local_id, local->msi_va); 63 return 0; 64 } 65 66 static void imsic_irq_compose_vector_msg(struct imsic_vector *vec, struct msi_msg *msg) 67 { 68 phys_addr_t msi_addr; 69 70 if (WARN_ON(!vec)) 71 return; 72 73 if (WARN_ON(!imsic_cpu_page_phys(vec->cpu, 0, &msi_addr))) 74 return; 75 76 msg->address_hi = upper_32_bits(msi_addr); 77 msg->address_lo = lower_32_bits(msi_addr); 78 msg->data = vec->local_id; 79 } 80 81 static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *msg) 82 { 83 imsic_irq_compose_vector_msg(irq_data_get_irq_chip_data(d), msg); 84 } 85 86 #ifdef CONFIG_SMP 87 static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vector *vec) 88 { 89 struct msi_msg msg = { }; 90 91 imsic_irq_compose_vector_msg(vec, &msg); 92 irq_data_get_irq_chip(d)->irq_write_msi_msg(d, &msg); 93 } 94 95 static int imsic_irq_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 96 bool force) 97 { 98 struct imsic_vector *old_vec, *new_vec; 99 struct irq_data *pd = d->parent_data; 100 101 old_vec = irq_data_get_irq_chip_data(pd); 102 if (WARN_ON(!old_vec)) 103 return -ENOENT; 104 105 /* If old vector cpu belongs to the target cpumask then do nothing */ 106 if (cpumask_test_cpu(old_vec->cpu, mask_val)) 107 return IRQ_SET_MASK_OK_DONE; 108 109 /* If move is already in-flight then return failure */ 110 if (imsic_vector_get_move(old_vec)) 111 return -EBUSY; 112 113 /* Get a new vector on the desired set of CPUs */ 114 new_vec = imsic_vector_alloc(old_vec->hwirq, mask_val); 115 if (!new_vec) 116 return -ENOSPC; 117 118 /* Point device to the new vector */ 119 imsic_msi_update_msg(d, new_vec); 120 121 /* Update irq descriptors with the new vector */ 122 pd->chip_data = new_vec; 123 124 /* Update effective affinity of parent irq data */ 125 irq_data_update_effective_affinity(pd, cpumask_of(new_vec->cpu)); 126 127 /* Move state of the old vector to the new vector */ 128 imsic_vector_move(old_vec, new_vec); 129 130 return IRQ_SET_MASK_OK_DONE; 131 } 132 #endif 133 134 static struct irq_chip imsic_irq_base_chip = { 135 .name = "IMSIC", 136 .irq_mask = imsic_irq_mask, 137 .irq_unmask = imsic_irq_unmask, 138 .irq_retrigger = imsic_irq_retrigger, 139 .irq_compose_msi_msg = imsic_irq_compose_msg, 140 .flags = IRQCHIP_SKIP_SET_WAKE | 141 IRQCHIP_MASK_ON_SUSPEND, 142 }; 143 144 static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 145 unsigned int nr_irqs, void *args) 146 { 147 struct imsic_vector *vec; 148 149 /* Multi-MSI is not supported yet. */ 150 if (nr_irqs > 1) 151 return -EOPNOTSUPP; 152 153 vec = imsic_vector_alloc(virq, cpu_online_mask); 154 if (!vec) 155 return -ENOSPC; 156 157 irq_domain_set_info(domain, virq, virq, &imsic_irq_base_chip, vec, 158 handle_simple_irq, NULL, NULL); 159 irq_set_noprobe(virq); 160 irq_set_affinity(virq, cpu_online_mask); 161 irq_data_update_effective_affinity(irq_get_irq_data(virq), cpumask_of(vec->cpu)); 162 163 return 0; 164 } 165 166 static void imsic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 167 unsigned int nr_irqs) 168 { 169 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 170 171 imsic_vector_free(irq_data_get_irq_chip_data(d)); 172 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 173 } 174 175 static int imsic_irq_domain_select(struct irq_domain *domain, struct irq_fwspec *fwspec, 176 enum irq_domain_bus_token bus_token) 177 { 178 const struct msi_parent_ops *ops = domain->msi_parent_ops; 179 u32 busmask = BIT(bus_token); 180 181 if (fwspec->fwnode != domain->fwnode || fwspec->param_count != 0) 182 return 0; 183 184 /* Handle pure domain searches */ 185 if (bus_token == ops->bus_select_token) 186 return 1; 187 188 return !!(ops->bus_select_mask & busmask); 189 } 190 191 #ifdef CONFIG_GENERIC_IRQ_DEBUGFS 192 static void imsic_irq_debug_show(struct seq_file *m, struct irq_domain *d, 193 struct irq_data *irqd, int ind) 194 { 195 if (!irqd) { 196 imsic_vector_debug_show_summary(m, ind); 197 return; 198 } 199 200 imsic_vector_debug_show(m, irq_data_get_irq_chip_data(irqd), ind); 201 } 202 #endif 203 204 static const struct irq_domain_ops imsic_base_domain_ops = { 205 .alloc = imsic_irq_domain_alloc, 206 .free = imsic_irq_domain_free, 207 .select = imsic_irq_domain_select, 208 #ifdef CONFIG_GENERIC_IRQ_DEBUGFS 209 .debug_show = imsic_irq_debug_show, 210 #endif 211 }; 212 213 #ifdef CONFIG_RISCV_IMSIC_PCI 214 215 static void imsic_pci_mask_irq(struct irq_data *d) 216 { 217 pci_msi_mask_irq(d); 218 irq_chip_mask_parent(d); 219 } 220 221 static void imsic_pci_unmask_irq(struct irq_data *d) 222 { 223 irq_chip_unmask_parent(d); 224 pci_msi_unmask_irq(d); 225 } 226 227 #define MATCH_PCI_MSI BIT(DOMAIN_BUS_PCI_MSI) 228 229 #else 230 231 #define MATCH_PCI_MSI 0 232 233 #endif 234 235 static bool imsic_init_dev_msi_info(struct device *dev, 236 struct irq_domain *domain, 237 struct irq_domain *real_parent, 238 struct msi_domain_info *info) 239 { 240 const struct msi_parent_ops *pops = real_parent->msi_parent_ops; 241 242 /* MSI parent domain specific settings */ 243 switch (real_parent->bus_token) { 244 case DOMAIN_BUS_NEXUS: 245 if (WARN_ON_ONCE(domain != real_parent)) 246 return false; 247 #ifdef CONFIG_SMP 248 info->chip->irq_set_affinity = imsic_irq_set_affinity; 249 #endif 250 break; 251 default: 252 WARN_ON_ONCE(1); 253 return false; 254 } 255 256 /* Is the target supported? */ 257 switch (info->bus_token) { 258 #ifdef CONFIG_RISCV_IMSIC_PCI 259 case DOMAIN_BUS_PCI_DEVICE_MSI: 260 case DOMAIN_BUS_PCI_DEVICE_MSIX: 261 info->chip->irq_mask = imsic_pci_mask_irq; 262 info->chip->irq_unmask = imsic_pci_unmask_irq; 263 break; 264 #endif 265 case DOMAIN_BUS_DEVICE_MSI: 266 /* 267 * Per-device MSI should never have any MSI feature bits 268 * set. It's sole purpose is to create a dumb interrupt 269 * chip which has a device specific irq_write_msi_msg() 270 * callback. 271 */ 272 if (WARN_ON_ONCE(info->flags)) 273 return false; 274 275 /* Core managed MSI descriptors */ 276 info->flags |= MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | 277 MSI_FLAG_FREE_MSI_DESCS; 278 break; 279 case DOMAIN_BUS_WIRED_TO_MSI: 280 break; 281 default: 282 WARN_ON_ONCE(1); 283 return false; 284 } 285 286 /* Use hierarchial chip operations re-trigger */ 287 info->chip->irq_retrigger = irq_chip_retrigger_hierarchy; 288 289 /* 290 * Mask out the domain specific MSI feature flags which are not 291 * supported by the real parent. 292 */ 293 info->flags &= pops->supported_flags; 294 295 /* Enforce the required flags */ 296 info->flags |= pops->required_flags; 297 298 return true; 299 } 300 301 #define MATCH_PLATFORM_MSI BIT(DOMAIN_BUS_PLATFORM_MSI) 302 303 static const struct msi_parent_ops imsic_msi_parent_ops = { 304 .supported_flags = MSI_GENERIC_FLAGS_MASK | 305 MSI_FLAG_PCI_MSIX, 306 .required_flags = MSI_FLAG_USE_DEF_DOM_OPS | 307 MSI_FLAG_USE_DEF_CHIP_OPS, 308 .bus_select_token = DOMAIN_BUS_NEXUS, 309 .bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI, 310 .init_dev_msi_info = imsic_init_dev_msi_info, 311 }; 312 313 int imsic_irqdomain_init(void) 314 { 315 struct imsic_global_config *global; 316 317 if (!imsic || !imsic->fwnode) { 318 pr_err("early driver not probed\n"); 319 return -ENODEV; 320 } 321 322 if (imsic->base_domain) { 323 pr_err("%pfwP: irq domain already created\n", imsic->fwnode); 324 return -ENODEV; 325 } 326 327 /* Create Base IRQ domain */ 328 imsic->base_domain = irq_domain_create_tree(imsic->fwnode, 329 &imsic_base_domain_ops, imsic); 330 if (!imsic->base_domain) { 331 pr_err("%pfwP: failed to create IMSIC base domain\n", imsic->fwnode); 332 return -ENOMEM; 333 } 334 imsic->base_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT; 335 imsic->base_domain->msi_parent_ops = &imsic_msi_parent_ops; 336 337 irq_domain_update_bus_token(imsic->base_domain, DOMAIN_BUS_NEXUS); 338 339 global = &imsic->global; 340 pr_info("%pfwP: hart-index-bits: %d, guest-index-bits: %d\n", 341 imsic->fwnode, global->hart_index_bits, global->guest_index_bits); 342 pr_info("%pfwP: group-index-bits: %d, group-index-shift: %d\n", 343 imsic->fwnode, global->group_index_bits, global->group_index_shift); 344 pr_info("%pfwP: per-CPU IDs %d at base address %pa\n", 345 imsic->fwnode, global->nr_ids, &global->base_addr); 346 pr_info("%pfwP: total %d interrupts available\n", 347 imsic->fwnode, num_possible_cpus() * (global->nr_ids - 1)); 348 349 return 0; 350 } 351 352 static int imsic_platform_probe_common(struct fwnode_handle *fwnode) 353 { 354 if (imsic && imsic->fwnode != fwnode) { 355 pr_err("%pfwP: fwnode mismatch\n", fwnode); 356 return -ENODEV; 357 } 358 359 return imsic_irqdomain_init(); 360 } 361 362 static int imsic_platform_dt_probe(struct platform_device *pdev) 363 { 364 return imsic_platform_probe_common(pdev->dev.fwnode); 365 } 366 367 #ifdef CONFIG_ACPI 368 369 /* 370 * On ACPI based systems, PCI enumeration happens early during boot in 371 * acpi_scan_init(). PCI enumeration expects MSI domain setup before 372 * it calls pci_set_msi_domain(). Hence, unlike in DT where 373 * imsic-platform drive probe happens late during boot, ACPI based 374 * systems need to setup the MSI domain early. 375 */ 376 int imsic_platform_acpi_probe(struct fwnode_handle *fwnode) 377 { 378 return imsic_platform_probe_common(fwnode); 379 } 380 381 #endif 382 383 static const struct of_device_id imsic_platform_match[] = { 384 { .compatible = "riscv,imsics" }, 385 {} 386 }; 387 388 static struct platform_driver imsic_platform_driver = { 389 .driver = { 390 .name = "riscv-imsic", 391 .of_match_table = imsic_platform_match, 392 }, 393 .probe = imsic_platform_dt_probe, 394 }; 395 builtin_platform_driver(imsic_platform_driver); 396