1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2021 Western Digital Corporation or its affiliates. 4 * Copyright (C) 2022 Ventana Micro Systems Inc. 5 */ 6 7 #define pr_fmt(fmt) "riscv-imsic: " fmt 8 #include <linux/bitmap.h> 9 #include <linux/cpu.h> 10 #include <linux/interrupt.h> 11 #include <linux/io.h> 12 #include <linux/irq.h> 13 #include <linux/irqchip.h> 14 #include <linux/irqdomain.h> 15 #include <linux/module.h> 16 #include <linux/msi.h> 17 #include <linux/pci.h> 18 #include <linux/platform_device.h> 19 #include <linux/spinlock.h> 20 #include <linux/smp.h> 21 22 #include "irq-riscv-imsic-state.h" 23 24 static bool imsic_cpu_page_phys(unsigned int cpu, unsigned int guest_index, 25 phys_addr_t *out_msi_pa) 26 { 27 struct imsic_global_config *global; 28 struct imsic_local_config *local; 29 30 global = &imsic->global; 31 local = per_cpu_ptr(global->local, cpu); 32 33 if (BIT(global->guest_index_bits) <= guest_index) 34 return false; 35 36 if (out_msi_pa) 37 *out_msi_pa = local->msi_pa + (guest_index * IMSIC_MMIO_PAGE_SZ); 38 39 return true; 40 } 41 42 static void imsic_irq_mask(struct irq_data *d) 43 { 44 imsic_vector_mask(irq_data_get_irq_chip_data(d)); 45 } 46 47 static void imsic_irq_unmask(struct irq_data *d) 48 { 49 imsic_vector_unmask(irq_data_get_irq_chip_data(d)); 50 } 51 52 static int imsic_irq_retrigger(struct irq_data *d) 53 { 54 struct imsic_vector *vec = irq_data_get_irq_chip_data(d); 55 struct imsic_local_config *local; 56 57 if (WARN_ON(!vec)) 58 return -ENOENT; 59 60 local = per_cpu_ptr(imsic->global.local, vec->cpu); 61 writel_relaxed(vec->local_id, local->msi_va); 62 return 0; 63 } 64 65 static void imsic_irq_compose_vector_msg(struct imsic_vector *vec, struct msi_msg *msg) 66 { 67 phys_addr_t msi_addr; 68 69 if (WARN_ON(!vec)) 70 return; 71 72 if (WARN_ON(!imsic_cpu_page_phys(vec->cpu, 0, &msi_addr))) 73 return; 74 75 msg->address_hi = upper_32_bits(msi_addr); 76 msg->address_lo = lower_32_bits(msi_addr); 77 msg->data = vec->local_id; 78 } 79 80 static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *msg) 81 { 82 imsic_irq_compose_vector_msg(irq_data_get_irq_chip_data(d), msg); 83 } 84 85 #ifdef CONFIG_SMP 86 static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vector *vec) 87 { 88 struct msi_msg msg = { }; 89 90 imsic_irq_compose_vector_msg(vec, &msg); 91 irq_data_get_irq_chip(d)->irq_write_msi_msg(d, &msg); 92 } 93 94 static int imsic_irq_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 95 bool force) 96 { 97 struct imsic_vector *old_vec, *new_vec; 98 struct irq_data *pd = d->parent_data; 99 100 old_vec = irq_data_get_irq_chip_data(pd); 101 if (WARN_ON(!old_vec)) 102 return -ENOENT; 103 104 /* If old vector cpu belongs to the target cpumask then do nothing */ 105 if (cpumask_test_cpu(old_vec->cpu, mask_val)) 106 return IRQ_SET_MASK_OK_DONE; 107 108 /* If move is already in-flight then return failure */ 109 if (imsic_vector_get_move(old_vec)) 110 return -EBUSY; 111 112 /* Get a new vector on the desired set of CPUs */ 113 new_vec = imsic_vector_alloc(old_vec->hwirq, mask_val); 114 if (!new_vec) 115 return -ENOSPC; 116 117 /* Point device to the new vector */ 118 imsic_msi_update_msg(d, new_vec); 119 120 /* Update irq descriptors with the new vector */ 121 pd->chip_data = new_vec; 122 123 /* Update effective affinity of parent irq data */ 124 irq_data_update_effective_affinity(pd, cpumask_of(new_vec->cpu)); 125 126 /* Move state of the old vector to the new vector */ 127 imsic_vector_move(old_vec, new_vec); 128 129 return IRQ_SET_MASK_OK_DONE; 130 } 131 #endif 132 133 static struct irq_chip imsic_irq_base_chip = { 134 .name = "IMSIC", 135 .irq_mask = imsic_irq_mask, 136 .irq_unmask = imsic_irq_unmask, 137 .irq_retrigger = imsic_irq_retrigger, 138 .irq_compose_msi_msg = imsic_irq_compose_msg, 139 .flags = IRQCHIP_SKIP_SET_WAKE | 140 IRQCHIP_MASK_ON_SUSPEND, 141 }; 142 143 static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 144 unsigned int nr_irqs, void *args) 145 { 146 struct imsic_vector *vec; 147 148 /* Multi-MSI is not supported yet. */ 149 if (nr_irqs > 1) 150 return -EOPNOTSUPP; 151 152 vec = imsic_vector_alloc(virq, cpu_online_mask); 153 if (!vec) 154 return -ENOSPC; 155 156 irq_domain_set_info(domain, virq, virq, &imsic_irq_base_chip, vec, 157 handle_simple_irq, NULL, NULL); 158 irq_set_noprobe(virq); 159 irq_set_affinity(virq, cpu_online_mask); 160 irq_data_update_effective_affinity(irq_get_irq_data(virq), cpumask_of(vec->cpu)); 161 162 return 0; 163 } 164 165 static void imsic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 166 unsigned int nr_irqs) 167 { 168 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 169 170 imsic_vector_free(irq_data_get_irq_chip_data(d)); 171 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 172 } 173 174 static int imsic_irq_domain_select(struct irq_domain *domain, struct irq_fwspec *fwspec, 175 enum irq_domain_bus_token bus_token) 176 { 177 const struct msi_parent_ops *ops = domain->msi_parent_ops; 178 u32 busmask = BIT(bus_token); 179 180 if (fwspec->fwnode != domain->fwnode || fwspec->param_count != 0) 181 return 0; 182 183 /* Handle pure domain searches */ 184 if (bus_token == ops->bus_select_token) 185 return 1; 186 187 return !!(ops->bus_select_mask & busmask); 188 } 189 190 #ifdef CONFIG_GENERIC_IRQ_DEBUGFS 191 static void imsic_irq_debug_show(struct seq_file *m, struct irq_domain *d, 192 struct irq_data *irqd, int ind) 193 { 194 if (!irqd) { 195 imsic_vector_debug_show_summary(m, ind); 196 return; 197 } 198 199 imsic_vector_debug_show(m, irq_data_get_irq_chip_data(irqd), ind); 200 } 201 #endif 202 203 static const struct irq_domain_ops imsic_base_domain_ops = { 204 .alloc = imsic_irq_domain_alloc, 205 .free = imsic_irq_domain_free, 206 .select = imsic_irq_domain_select, 207 #ifdef CONFIG_GENERIC_IRQ_DEBUGFS 208 .debug_show = imsic_irq_debug_show, 209 #endif 210 }; 211 212 #ifdef CONFIG_RISCV_IMSIC_PCI 213 214 static void imsic_pci_mask_irq(struct irq_data *d) 215 { 216 pci_msi_mask_irq(d); 217 irq_chip_mask_parent(d); 218 } 219 220 static void imsic_pci_unmask_irq(struct irq_data *d) 221 { 222 irq_chip_unmask_parent(d); 223 pci_msi_unmask_irq(d); 224 } 225 226 #define MATCH_PCI_MSI BIT(DOMAIN_BUS_PCI_MSI) 227 228 #else 229 230 #define MATCH_PCI_MSI 0 231 232 #endif 233 234 static bool imsic_init_dev_msi_info(struct device *dev, 235 struct irq_domain *domain, 236 struct irq_domain *real_parent, 237 struct msi_domain_info *info) 238 { 239 const struct msi_parent_ops *pops = real_parent->msi_parent_ops; 240 241 /* MSI parent domain specific settings */ 242 switch (real_parent->bus_token) { 243 case DOMAIN_BUS_NEXUS: 244 if (WARN_ON_ONCE(domain != real_parent)) 245 return false; 246 #ifdef CONFIG_SMP 247 info->chip->irq_set_affinity = imsic_irq_set_affinity; 248 #endif 249 break; 250 default: 251 WARN_ON_ONCE(1); 252 return false; 253 } 254 255 /* Is the target supported? */ 256 switch (info->bus_token) { 257 #ifdef CONFIG_RISCV_IMSIC_PCI 258 case DOMAIN_BUS_PCI_DEVICE_MSI: 259 case DOMAIN_BUS_PCI_DEVICE_MSIX: 260 info->chip->irq_mask = imsic_pci_mask_irq; 261 info->chip->irq_unmask = imsic_pci_unmask_irq; 262 break; 263 #endif 264 case DOMAIN_BUS_DEVICE_MSI: 265 /* 266 * Per-device MSI should never have any MSI feature bits 267 * set. It's sole purpose is to create a dumb interrupt 268 * chip which has a device specific irq_write_msi_msg() 269 * callback. 270 */ 271 if (WARN_ON_ONCE(info->flags)) 272 return false; 273 274 /* Core managed MSI descriptors */ 275 info->flags |= MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | 276 MSI_FLAG_FREE_MSI_DESCS; 277 break; 278 case DOMAIN_BUS_WIRED_TO_MSI: 279 break; 280 default: 281 WARN_ON_ONCE(1); 282 return false; 283 } 284 285 /* Use hierarchial chip operations re-trigger */ 286 info->chip->irq_retrigger = irq_chip_retrigger_hierarchy; 287 288 /* 289 * Mask out the domain specific MSI feature flags which are not 290 * supported by the real parent. 291 */ 292 info->flags &= pops->supported_flags; 293 294 /* Enforce the required flags */ 295 info->flags |= pops->required_flags; 296 297 return true; 298 } 299 300 #define MATCH_PLATFORM_MSI BIT(DOMAIN_BUS_PLATFORM_MSI) 301 302 static const struct msi_parent_ops imsic_msi_parent_ops = { 303 .supported_flags = MSI_GENERIC_FLAGS_MASK | 304 MSI_FLAG_PCI_MSIX, 305 .required_flags = MSI_FLAG_USE_DEF_DOM_OPS | 306 MSI_FLAG_USE_DEF_CHIP_OPS, 307 .bus_select_token = DOMAIN_BUS_NEXUS, 308 .bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI, 309 .init_dev_msi_info = imsic_init_dev_msi_info, 310 }; 311 312 int imsic_irqdomain_init(void) 313 { 314 struct imsic_global_config *global; 315 316 if (!imsic || !imsic->fwnode) { 317 pr_err("early driver not probed\n"); 318 return -ENODEV; 319 } 320 321 if (imsic->base_domain) { 322 pr_err("%pfwP: irq domain already created\n", imsic->fwnode); 323 return -ENODEV; 324 } 325 326 /* Create Base IRQ domain */ 327 imsic->base_domain = irq_domain_create_tree(imsic->fwnode, 328 &imsic_base_domain_ops, imsic); 329 if (!imsic->base_domain) { 330 pr_err("%pfwP: failed to create IMSIC base domain\n", imsic->fwnode); 331 return -ENOMEM; 332 } 333 imsic->base_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT; 334 imsic->base_domain->msi_parent_ops = &imsic_msi_parent_ops; 335 336 irq_domain_update_bus_token(imsic->base_domain, DOMAIN_BUS_NEXUS); 337 338 global = &imsic->global; 339 pr_info("%pfwP: hart-index-bits: %d, guest-index-bits: %d\n", 340 imsic->fwnode, global->hart_index_bits, global->guest_index_bits); 341 pr_info("%pfwP: group-index-bits: %d, group-index-shift: %d\n", 342 imsic->fwnode, global->group_index_bits, global->group_index_shift); 343 pr_info("%pfwP: per-CPU IDs %d at base PPN %pa\n", 344 imsic->fwnode, global->nr_ids, &global->base_addr); 345 pr_info("%pfwP: total %d interrupts available\n", 346 imsic->fwnode, num_possible_cpus() * (global->nr_ids - 1)); 347 348 return 0; 349 } 350 351 static int imsic_platform_probe(struct platform_device *pdev) 352 { 353 struct device *dev = &pdev->dev; 354 355 if (imsic && imsic->fwnode != dev->fwnode) { 356 dev_err(dev, "fwnode mismatch\n"); 357 return -ENODEV; 358 } 359 360 return imsic_irqdomain_init(); 361 } 362 363 static const struct of_device_id imsic_platform_match[] = { 364 { .compatible = "riscv,imsics" }, 365 {} 366 }; 367 368 static struct platform_driver imsic_platform_driver = { 369 .driver = { 370 .name = "riscv-imsic", 371 .of_match_table = imsic_platform_match, 372 }, 373 .probe = imsic_platform_probe, 374 }; 375 builtin_platform_driver(imsic_platform_driver); 376